1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/Proposal/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanHCFGBuilder.h"
61#include "VPlanPredicator.h"
62#include "VPlanTransforms.h"
63#include "llvm/ADT/APInt.h"
64#include "llvm/ADT/ArrayRef.h"
65#include "llvm/ADT/DenseMap.h"
66#include "llvm/ADT/DenseMapInfo.h"
67#include "llvm/ADT/Hashing.h"
68#include "llvm/ADT/MapVector.h"
69#include "llvm/ADT/None.h"
70#include "llvm/ADT/Optional.h"
71#include "llvm/ADT/STLExtras.h"
72#include "llvm/ADT/SetVector.h"
73#include "llvm/ADT/SmallPtrSet.h"
74#include "llvm/ADT/SmallVector.h"
75#include "llvm/ADT/Statistic.h"
76#include "llvm/ADT/StringRef.h"
77#include "llvm/ADT/Twine.h"
78#include "llvm/ADT/iterator_range.h"
79#include "llvm/Analysis/AssumptionCache.h"
80#include "llvm/Analysis/BasicAliasAnalysis.h"
81#include "llvm/Analysis/BlockFrequencyInfo.h"
82#include "llvm/Analysis/CFG.h"
83#include "llvm/Analysis/CodeMetrics.h"
84#include "llvm/Analysis/DemandedBits.h"
85#include "llvm/Analysis/GlobalsModRef.h"
86#include "llvm/Analysis/LoopAccessAnalysis.h"
87#include "llvm/Analysis/LoopAnalysisManager.h"
88#include "llvm/Analysis/LoopInfo.h"
89#include "llvm/Analysis/LoopIterator.h"
90#include "llvm/Analysis/MemorySSA.h"
91#include "llvm/Analysis/OptimizationRemarkEmitter.h"
92#include "llvm/Analysis/ProfileSummaryInfo.h"
93#include "llvm/Analysis/ScalarEvolution.h"
94#include "llvm/Analysis/ScalarEvolutionExpressions.h"
95#include "llvm/Analysis/TargetLibraryInfo.h"
96#include "llvm/Analysis/TargetTransformInfo.h"
97#include "llvm/Analysis/VectorUtils.h"
98#include "llvm/IR/Attributes.h"
99#include "llvm/IR/BasicBlock.h"
100#include "llvm/IR/CFG.h"
101#include "llvm/IR/Constant.h"
102#include "llvm/IR/Constants.h"
103#include "llvm/IR/DataLayout.h"
104#include "llvm/IR/DebugInfoMetadata.h"
105#include "llvm/IR/DebugLoc.h"
106#include "llvm/IR/DerivedTypes.h"
107#include "llvm/IR/DiagnosticInfo.h"
108#include "llvm/IR/Dominators.h"
109#include "llvm/IR/Function.h"
110#include "llvm/IR/IRBuilder.h"
111#include "llvm/IR/InstrTypes.h"
112#include "llvm/IR/Instruction.h"
113#include "llvm/IR/Instructions.h"
114#include "llvm/IR/IntrinsicInst.h"
115#include "llvm/IR/Intrinsics.h"
116#include "llvm/IR/LLVMContext.h"
117#include "llvm/IR/Metadata.h"
118#include "llvm/IR/Module.h"
119#include "llvm/IR/Operator.h"
120#include "llvm/IR/Type.h"
121#include "llvm/IR/Use.h"
122#include "llvm/IR/User.h"
123#include "llvm/IR/Value.h"
124#include "llvm/IR/ValueHandle.h"
125#include "llvm/IR/Verifier.h"
126#include "llvm/InitializePasses.h"
127#include "llvm/Pass.h"
128#include "llvm/Support/Casting.h"
129#include "llvm/Support/CommandLine.h"
130#include "llvm/Support/Compiler.h"
131#include "llvm/Support/Debug.h"
132#include "llvm/Support/ErrorHandling.h"
133#include "llvm/Support/InstructionCost.h"
134#include "llvm/Support/MathExtras.h"
135#include "llvm/Support/raw_ostream.h"
136#include "llvm/Transforms/Utils/BasicBlockUtils.h"
137#include "llvm/Transforms/Utils/InjectTLIMappings.h"
138#include "llvm/Transforms/Utils/LoopSimplify.h"
139#include "llvm/Transforms/Utils/LoopUtils.h"
140#include "llvm/Transforms/Utils/LoopVersioning.h"
141#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
142#include "llvm/Transforms/Utils/SizeOpts.h"
143#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
144#include <algorithm>
145#include <cassert>
146#include <cstdint>
147#include <cstdlib>
148#include <functional>
149#include <iterator>
150#include <limits>
151#include <memory>
152#include <string>
153#include <tuple>
154#include <utility>
155
156using namespace llvm;
157
158#define LV_NAME "loop-vectorize"
159#define DEBUG_TYPE LV_NAME
160
161#ifndef NDEBUG
162const char VerboseDebug[] = DEBUG_TYPE "-verbose";
163#endif
164
165/// @{
166/// Metadata attribute names
167const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all";
168const char LLVMLoopVectorizeFollowupVectorized[] =
169 "llvm.loop.vectorize.followup_vectorized";
170const char LLVMLoopVectorizeFollowupEpilogue[] =
171 "llvm.loop.vectorize.followup_epilogue";
172/// @}
173
174STATISTIC(LoopsVectorized, "Number of loops vectorized");
175STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
176STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
177
178static cl::opt<bool> EnableEpilogueVectorization(
179 "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
180 cl::desc("Enable vectorization of epilogue loops."));
181
182static cl::opt<unsigned> EpilogueVectorizationForceVF(
183 "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
184 cl::desc("When epilogue vectorization is enabled, and a value greater than "
185 "1 is specified, forces the given VF for all applicable epilogue "
186 "loops."));
187
188static cl::opt<unsigned> EpilogueVectorizationMinVF(
189 "epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden,
190 cl::desc("Only loops with vectorization factor equal to or larger than "
191 "the specified value are considered for epilogue vectorization."));
192
193/// Loops with a known constant trip count below this number are vectorized only
194/// if no scalar iteration overheads are incurred.
195static cl::opt<unsigned> TinyTripCountVectorThreshold(
196 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
197 cl::desc("Loops with a constant trip count that is smaller than this "
198 "value are vectorized only if no scalar iteration overheads "
199 "are incurred."));
200
201// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
202// that predication is preferred, and this lists all options. I.e., the
203// vectorizer will try to fold the tail-loop (epilogue) into the vector body
204// and predicate the instructions accordingly. If tail-folding fails, there are
205// different fallback strategies depending on these values:
206namespace PreferPredicateTy {
207 enum Option {
208 ScalarEpilogue = 0,
209 PredicateElseScalarEpilogue,
210 PredicateOrDontVectorize
211 };
212} // namespace PreferPredicateTy
213
214static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
215 "prefer-predicate-over-epilogue",
216 cl::init(PreferPredicateTy::ScalarEpilogue),
217 cl::Hidden,
218 cl::desc("Tail-folding and predication preferences over creating a scalar "
219 "epilogue loop."),
220 cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,
221 "scalar-epilogue",
222 "Don't tail-predicate loops, create scalar epilogue"),
223 clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
224 "predicate-else-scalar-epilogue",
225 "prefer tail-folding, create scalar epilogue if tail "
226 "folding fails."),
227 clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
228 "predicate-dont-vectorize",
229 "prefers tail-folding, don't attempt vectorization if "
230 "tail-folding fails.")));
231
232static cl::opt<bool> MaximizeBandwidth(
233 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
234 cl::desc("Maximize bandwidth when selecting vectorization factor which "
235 "will be determined by the smallest type in loop."));
236
237static cl::opt<bool> EnableInterleavedMemAccesses(
238 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
239 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
240
241/// An interleave-group may need masking if it resides in a block that needs
242/// predication, or in order to mask away gaps.
243static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
244 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
245 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
246
247static cl::opt<unsigned> TinyTripCountInterleaveThreshold(
248 "tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden,
249 cl::desc("We don't interleave loops with a estimated constant trip count "
250 "below this number"));
251
252static cl::opt<unsigned> ForceTargetNumScalarRegs(
253 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
254 cl::desc("A flag that overrides the target's number of scalar registers."));
255
256static cl::opt<unsigned> ForceTargetNumVectorRegs(
257 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
258 cl::desc("A flag that overrides the target's number of vector registers."));
259
260static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
261 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
262 cl::desc("A flag that overrides the target's max interleave factor for "
263 "scalar loops."));
264
265static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
266 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
267 cl::desc("A flag that overrides the target's max interleave factor for "
268 "vectorized loops."));
269
270static cl::opt<unsigned> ForceTargetInstructionCost(
271 "force-target-instruction-cost", cl::init(0), cl::Hidden,
272 cl::desc("A flag that overrides the target's expected cost for "
273 "an instruction to a single constant value. Mostly "
274 "useful for getting consistent testing."));
275
276static cl::opt<bool> ForceTargetSupportsScalableVectors(
277 "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
278 cl::desc(
279 "Pretend that scalable vectors are supported, even if the target does "
280 "not support them. This flag should only be used for testing."));
281
282static cl::opt<unsigned> SmallLoopCost(
283 "small-loop-cost", cl::init(20), cl::Hidden,
284 cl::desc(
285 "The cost of a loop that is considered 'small' by the interleaver."));
286
287static cl::opt<bool> LoopVectorizeWithBlockFrequency(
288 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
289 cl::desc("Enable the use of the block frequency analysis to access PGO "
290 "heuristics minimizing code growth in cold regions and being more "
291 "aggressive in hot regions."));
292
293// Runtime interleave loops for load/store throughput.
294static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
295 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
296 cl::desc(
297 "Enable runtime interleaving until load/store ports are saturated"));
298
299/// Interleave small loops with scalar reductions.
300static cl::opt<bool> InterleaveSmallLoopScalarReduction(
301 "interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden,
302 cl::desc("Enable interleaving for loops with small iteration counts that "
303 "contain scalar reductions to expose ILP."));
304
305/// The number of stores in a loop that are allowed to need predication.
306static cl::opt<unsigned> NumberOfStoresToPredicate(
307 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
308 cl::desc("Max number of stores to be predicated behind an if."));
309
310static cl::opt<bool> EnableIndVarRegisterHeur(
311 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
312 cl::desc("Count the induction variable only once when interleaving"));
313
314static cl::opt<bool> EnableCondStoresVectorization(
315 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
316 cl::desc("Enable if predication of stores during vectorization."));
317
318static cl::opt<unsigned> MaxNestedScalarReductionIC(
319 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
320 cl::desc("The maximum interleave count to use when interleaving a scalar "
321 "reduction in a nested loop."));
322
323static cl::opt<bool>
324 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
325 cl::Hidden,
326 cl::desc("Prefer in-loop vector reductions, "
327 "overriding the targets preference."));
328
329static cl::opt<bool> PreferPredicatedReductionSelect(
330 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
331 cl::desc(
332 "Prefer predicating a reduction operation over an after loop select."));
333
334cl::opt<bool> EnableVPlanNativePath(
335 "enable-vplan-native-path", cl::init(false), cl::Hidden,
336 cl::desc("Enable VPlan-native vectorization path with "
337 "support for outer loop vectorization."));
338
339// FIXME: Remove this switch once we have divergence analysis. Currently we
340// assume divergent non-backedge branches when this switch is true.
341cl::opt<bool> EnableVPlanPredication(
342 "enable-vplan-predication", cl::init(false), cl::Hidden,
343 cl::desc("Enable VPlan-native vectorization path predicator with "
344 "support for outer loop vectorization."));
345
346// This flag enables the stress testing of the VPlan H-CFG construction in the
347// VPlan-native vectorization path. It must be used in conjuction with
348// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
349// verification of the H-CFGs built.
350static cl::opt<bool> VPlanBuildStressTest(
351 "vplan-build-stress-test", cl::init(false), cl::Hidden,
352 cl::desc(
353 "Build VPlan for every supported loop nest in the function and bail "
354 "out right after the build (stress test the VPlan H-CFG construction "
355 "in the VPlan-native vectorization path)."));
356
357cl::opt<bool> llvm::EnableLoopInterleaving(
358 "interleave-loops", cl::init(true), cl::Hidden,
359 cl::desc("Enable loop interleaving in Loop vectorization passes"));
360cl::opt<bool> llvm::EnableLoopVectorization(
361 "vectorize-loops", cl::init(true), cl::Hidden,
362 cl::desc("Run the Loop vectorization passes"));
363
364/// A helper function that returns the type of loaded or stored value.
365static Type *getMemInstValueType(Value *I) {
366 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
367 "Expected Load or Store instruction");
368 if (auto *LI = dyn_cast<LoadInst>(I))
369 return LI->getType();
370 return cast<StoreInst>(I)->getValueOperand()->getType();
371}
372
373/// A helper function that returns true if the given type is irregular. The
374/// type is irregular if its allocated size doesn't equal the store size of an
375/// element of the corresponding vector type.
376static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
377 // Determine if an array of N elements of type Ty is "bitcast compatible"
378 // with a <N x Ty> vector.
379 // This is only true if there is no padding between the array elements.
380 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
381}
382
383/// A helper function that returns the reciprocal of the block probability of
384/// predicated blocks. If we return X, we are assuming the predicated block
385/// will execute once for every X iterations of the loop header.
386///
387/// TODO: We should use actual block probability here, if available. Currently,
388/// we always assume predicated blocks have a 50% chance of executing.
389static unsigned getReciprocalPredBlockProb() { return 2; }
390
391/// A helper function that adds a 'fast' flag to floating-point operations.
392static Value *addFastMathFlag(Value *V) {
393 if (isa<FPMathOperator>(V))
394 cast<Instruction>(V)->setFastMathFlags(FastMathFlags::getFast());
395 return V;
396}
397
398static Value *addFastMathFlag(Value *V, FastMathFlags FMF) {
399 if (isa<FPMathOperator>(V))
400 cast<Instruction>(V)->setFastMathFlags(FMF);
401 return V;
402}
403
404/// A helper function that returns an integer or floating-point constant with
405/// value C.
406static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {
407 return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)
408 : ConstantFP::get(Ty, C);
409}
410
411/// Returns "best known" trip count for the specified loop \p L as defined by
412/// the following procedure:
413/// 1) Returns exact trip count if it is known.
414/// 2) Returns expected trip count according to profile data if any.
415/// 3) Returns upper bound estimate if it is known.
416/// 4) Returns None if all of the above failed.
417static Optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE, Loop *L) {
418 // Check if exact trip count is known.
419 if (unsigned ExpectedTC = SE.getSmallConstantTripCount(L))
420 return ExpectedTC;
421
422 // Check if there is an expected trip count available from profile data.
423 if (LoopVectorizeWithBlockFrequency)
424 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
425 return EstimatedTC;
426
427 // Check if upper bound estimate is known.
428 if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
429 return ExpectedTC;
430
431 return None;
432}
433
434namespace llvm {
435
436/// InnerLoopVectorizer vectorizes loops which contain only one basic
437/// block to a specified vectorization factor (VF).
438/// This class performs the widening of scalars into vectors, or multiple
439/// scalars. This class also implements the following features:
440/// * It inserts an epilogue loop for handling loops that don't have iteration
441/// counts that are known to be a multiple of the vectorization factor.
442/// * It handles the code generation for reduction variables.
443/// * Scalarization (implementation using scalars) of un-vectorizable
444/// instructions.
445/// InnerLoopVectorizer does not perform any vectorization-legality
446/// checks, and relies on the caller to check for the different legality
447/// aspects. The InnerLoopVectorizer relies on the
448/// LoopVectorizationLegality class to provide information about the induction
449/// and reduction variables that were found to a given vectorization factor.
450class InnerLoopVectorizer {
451public:
452 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
453 LoopInfo *LI, DominatorTree *DT,
454 const TargetLibraryInfo *TLI,
455 const TargetTransformInfo *TTI, AssumptionCache *AC,
456 OptimizationRemarkEmitter *ORE, ElementCount VecWidth,
457 unsigned UnrollFactor, LoopVectorizationLegality *LVL,
458 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
459 ProfileSummaryInfo *PSI)
460 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
461 AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
462 Builder(PSE.getSE()->getContext()),
463 VectorLoopValueMap(UnrollFactor, VecWidth), Legal(LVL), Cost(CM),
464 BFI(BFI), PSI(PSI) {
465 // Query this against the original loop and save it here because the profile
466 // of the original loop header may change as the transformation happens.
467 OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
468 OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
469 }
470
471 virtual ~InnerLoopVectorizer() = default;
472
473 /// Create a new empty loop that will contain vectorized instructions later
474 /// on, while the old loop will be used as the scalar remainder. Control flow
475 /// is generated around the vectorized (and scalar epilogue) loops consisting
476 /// of various checks and bypasses. Return the pre-header block of the new
477 /// loop.
478 /// In the case of epilogue vectorization, this function is overriden to
479 /// handle the more complex control flow around the loops.
480 virtual BasicBlock *createVectorizedLoopSkeleton();
481
482 /// Widen a single instruction within the innermost loop.
483 void widenInstruction(Instruction &I, VPValue *Def, VPUser &Operands,
484 VPTransformState &State);
485
486 /// Widen a single call instruction within the innermost loop.
487 void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands,
488 VPTransformState &State);
489
490 /// Widen a single select instruction within the innermost loop.
491 void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands,
492 bool InvariantCond, VPTransformState &State);
493
494 /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
495 void fixVectorizedLoop();
496
497 // Return true if any runtime check is added.
498 bool areSafetyChecksAdded() { return AddedSafetyChecks; }
499
500 /// A type for vectorized values in the new loop. Each value from the
501 /// original loop, when vectorized, is represented by UF vector values in the
502 /// new unrolled loop, where UF is the unroll factor.
503 using VectorParts = SmallVector<Value *, 2>;
504
505 /// Vectorize a single GetElementPtrInst based on information gathered and
506 /// decisions taken during planning.
507 void widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices,
508 unsigned UF, ElementCount VF, bool IsPtrLoopInvariant,
509 SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);
510
511 /// Vectorize a single PHINode in a block. This method handles the induction
512 /// variable canonicalization. It supports both VF = 1 for unrolled loops and
513 /// arbitrary length vectors.
514 void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc,
515 Value *StartV, unsigned UF, ElementCount VF);
516
517 /// A helper function to scalarize a single Instruction in the innermost loop.
518 /// Generates a sequence of scalar instances for each lane between \p MinLane
519 /// and \p MaxLane, times each part between \p MinPart and \p MaxPart,
520 /// inclusive. Uses the VPValue operands from \p Operands instead of \p
521 /// Instr's operands.
522 void scalarizeInstruction(Instruction *Instr, VPUser &Operands,
523 const VPIteration &Instance, bool IfPredicateInstr,
524 VPTransformState &State);
525
526 /// Widen an integer or floating-point induction variable \p IV. If \p Trunc
527 /// is provided, the integer induction variable will first be truncated to
528 /// the corresponding type.
529 void widenIntOrFpInduction(PHINode *IV, Value *Start,
530 TruncInst *Trunc = nullptr);
531
532 /// getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a
533 /// vector or scalar value on-demand if one is not yet available. When
534 /// vectorizing a loop, we visit the definition of an instruction before its
535 /// uses. When visiting the definition, we either vectorize or scalarize the
536 /// instruction, creating an entry for it in the corresponding map. (In some
537 /// cases, such as induction variables, we will create both vector and scalar
538 /// entries.) Then, as we encounter uses of the definition, we derive values
539 /// for each scalar or vector use unless such a value is already available.
540 /// For example, if we scalarize a definition and one of its uses is vector,
541 /// we build the required vector on-demand with an insertelement sequence
542 /// when visiting the use. Otherwise, if the use is scalar, we can use the
543 /// existing scalar definition.
544 ///
545 /// Return a value in the new loop corresponding to \p V from the original
546 /// loop at unroll index \p Part. If the value has already been vectorized,
547 /// the corresponding vector entry in VectorLoopValueMap is returned. If,
548 /// however, the value has a scalar entry in VectorLoopValueMap, we construct
549 /// a new vector value on-demand by inserting the scalar values into a vector
550 /// with an insertelement sequence. If the value has been neither vectorized
551 /// nor scalarized, it must be loop invariant, so we simply broadcast the
552 /// value into a vector.
553 Value *getOrCreateVectorValue(Value *V, unsigned Part);
554
555 void setVectorValue(Value *Scalar, unsigned Part, Value *Vector) {
556 VectorLoopValueMap.setVectorValue(Scalar, Part, Vector);
557 }
558
559 /// Return a value in the new loop corresponding to \p V from the original
560 /// loop at unroll and vector indices \p Instance. If the value has been
561 /// vectorized but not scalarized, the necessary extractelement instruction
562 /// will be generated.
563 Value *getOrCreateScalarValue(Value *V, const VPIteration &Instance);
564
565 /// Construct the vector value of a scalarized value \p V one lane at a time.
566 void packScalarIntoVectorValue(Value *V, const VPIteration &Instance);
567
568 /// Try to vectorize interleaved access group \p Group with the base address
569 /// given in \p Addr, optionally masking the vector operations if \p
570 /// BlockInMask is non-null. Use \p State to translate given VPValues to IR
571 /// values in the vectorized loop.
572 void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
573 ArrayRef<VPValue *> VPDefs,
574 VPTransformState &State, VPValue *Addr,
575 ArrayRef<VPValue *> StoredValues,
576 VPValue *BlockInMask = nullptr);
577
578 /// Vectorize Load and Store instructions with the base address given in \p
579 /// Addr, optionally masking the vector operations if \p BlockInMask is
580 /// non-null. Use \p State to translate given VPValues to IR values in the
581 /// vectorized loop.
582 void vectorizeMemoryInstruction(Instruction *Instr, VPTransformState &State,
583 VPValue *Def, VPValue *Addr,
584 VPValue *StoredValue, VPValue *BlockInMask);
585
586 /// Set the debug location in the builder using the debug location in
587 /// the instruction.
588 void setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr);
589
590 /// Fix the non-induction PHIs in the OrigPHIsToFix vector.
591 void fixNonInductionPHIs(void);
592
593protected:
594 friend class LoopVectorizationPlanner;
595
596 /// A small list of PHINodes.
597 using PhiVector = SmallVector<PHINode *, 4>;
598
599 /// A type for scalarized values in the new loop. Each value from the
600 /// original loop, when scalarized, is represented by UF x VF scalar values
601 /// in the new unrolled loop, where UF is the unroll factor and VF is the
602 /// vectorization factor.
603 using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;
604
605 /// Set up the values of the IVs correctly when exiting the vector loop.
606 void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
607 Value *CountRoundDown, Value *EndValue,
608 BasicBlock *MiddleBlock);
609
610 /// Create a new induction variable inside L.
611 PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,
612 Value *Step, Instruction *DL);
613
614 /// Handle all cross-iteration phis in the header.
615 void fixCrossIterationPHIs();
616
617 /// Fix a first-order recurrence. This is the second phase of vectorizing
618 /// this phi node.
619 void fixFirstOrderRecurrence(PHINode *Phi);
620
621 /// Fix a reduction cross-iteration phi. This is the second phase of
622 /// vectorizing this phi node.
623 void fixReduction(PHINode *Phi);
624
625 /// Clear NSW/NUW flags from reduction instructions if necessary.
626 void clearReductionWrapFlags(RecurrenceDescriptor &RdxDesc);
627
628 /// Fixup the LCSSA phi nodes in the unique exit block. This simply
629 /// means we need to add the appropriate incoming value from the middle
630 /// block as exiting edges from the scalar epilogue loop (if present) are
631 /// already in place, and we exit the vector loop exclusively to the middle
632 /// block.
633 void fixLCSSAPHIs();
634
635 /// Iteratively sink the scalarized operands of a predicated instruction into
636 /// the block that was created for it.
637 void sinkScalarOperands(Instruction *PredInst);
638
639 /// Shrinks vector element sizes to the smallest bitwidth they can be legally
640 /// represented as.
641 void truncateToMinimalBitwidths();
642
643 /// Create a broadcast instruction. This method generates a broadcast
644 /// instruction (shuffle) for loop invariant values and for the induction
645 /// value. If this is the induction variable then we extend it to N, N+1, ...
646 /// this is needed because each iteration in the loop corresponds to a SIMD
647 /// element.
648 virtual Value *getBroadcastInstrs(Value *V);
649
650 /// This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...)
651 /// to each vector element of Val. The sequence starts at StartIndex.
652 /// \p Opcode is relevant for FP induction variable.
653 virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step,
654 Instruction::BinaryOps Opcode =
655 Instruction::BinaryOpsEnd);
656
657 /// Compute scalar induction steps. \p ScalarIV is the scalar induction
658 /// variable on which to base the steps, \p Step is the size of the step, and
659 /// \p EntryVal is the value from the original loop that maps to the steps.
660 /// Note that \p EntryVal doesn't have to be an induction variable - it
661 /// can also be a truncate instruction.
662 void buildScalarSteps(Value *ScalarIV, Value *Step, Instruction *EntryVal,
663 const InductionDescriptor &ID);
664
665 /// Create a vector induction phi node based on an existing scalar one. \p
666 /// EntryVal is the value from the original loop that maps to the vector phi
667 /// node, and \p Step is the loop-invariant step. If \p EntryVal is a
668 /// truncate instruction, instead of widening the original IV, we widen a
669 /// version of the IV truncated to \p EntryVal's type.
670 void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,
671 Value *Step, Value *Start,
672 Instruction *EntryVal);
673
674 /// Returns true if an instruction \p I should be scalarized instead of
675 /// vectorized for the chosen vectorization factor.
676 bool shouldScalarizeInstruction(Instruction *I) const;
677
678 /// Returns true if we should generate a scalar version of \p IV.
679 bool needsScalarInduction(Instruction *IV) const;
680
681 /// If there is a cast involved in the induction variable \p ID, which should
682 /// be ignored in the vectorized loop body, this function records the
683 /// VectorLoopValue of the respective Phi also as the VectorLoopValue of the
684 /// cast. We had already proved that the casted Phi is equal to the uncasted
685 /// Phi in the vectorized loop (under a runtime guard), and therefore
686 /// there is no need to vectorize the cast - the same value can be used in the
687 /// vector loop for both the Phi and the cast.
688 /// If \p VectorLoopValue is a scalarized value, \p Lane is also specified,
689 /// Otherwise, \p VectorLoopValue is a widened/vectorized value.
690 ///
691 /// \p EntryVal is the value from the original loop that maps to the vector
692 /// phi node and is used to distinguish what is the IV currently being
693 /// processed - original one (if \p EntryVal is a phi corresponding to the
694 /// original IV) or the "newly-created" one based on the proof mentioned above
695 /// (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the
696 /// latter case \p EntryVal is a TruncInst and we must not record anything for
697 /// that IV, but it's error-prone to expect callers of this routine to care
698 /// about that, hence this explicit parameter.
699 void recordVectorLoopValueForInductionCast(const InductionDescriptor &ID,
700 const Instruction *EntryVal,
701 Value *VectorLoopValue,
702 unsigned Part,
703 unsigned Lane = UINT_MAX);
704
705 /// Generate a shuffle sequence that will reverse the vector Vec.
706 virtual Value *reverseVector(Value *Vec);
707
708 /// Returns (and creates if needed) the original loop trip count.
709 Value *getOrCreateTripCount(Loop *NewLoop);
710
711 /// Returns (and creates if needed) the trip count of the widened loop.
712 Value *getOrCreateVectorTripCount(Loop *NewLoop);
713
714 /// Returns a bitcasted value to the requested vector type.
715 /// Also handles bitcasts of vector<float> <-> vector<pointer> types.
716 Value *createBitOrPointerCast(Value *V, VectorType *DstVTy,
717 const DataLayout &DL);
718
719 /// Emit a bypass check to see if the vector trip count is zero, including if
720 /// it overflows.
721 void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);
722
723 /// Emit a bypass check to see if all of the SCEV assumptions we've
724 /// had to make are correct.
725 void emitSCEVChecks(Loop *L, BasicBlock *Bypass);
726
727 /// Emit bypass checks to check any memory assumptions we may have made.
728 void emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);
729
730 /// Compute the transformed value of Index at offset StartValue using step
731 /// StepValue.
732 /// For integer induction, returns StartValue + Index * StepValue.
733 /// For pointer induction, returns StartValue[Index * StepValue].
734 /// FIXME: The newly created binary instructions should contain nsw/nuw
735 /// flags, which can be found from the original scalar operations.
736 Value *emitTransformedIndex(IRBuilder<> &B, Value *Index, ScalarEvolution *SE,
737 const DataLayout &DL,
738 const InductionDescriptor &ID) const;
739
740 /// Emit basic blocks (prefixed with \p Prefix) for the iteration check,
741 /// vector loop preheader, middle block and scalar preheader. Also
742 /// allocate a loop object for the new vector loop and return it.
743 Loop *createVectorLoopSkeleton(StringRef Prefix);
744
745 /// Create new phi nodes for the induction variables to resume iteration count
746 /// in the scalar epilogue, from where the vectorized loop left off (given by
747 /// \p VectorTripCount).
748 /// In cases where the loop skeleton is more complicated (eg. epilogue
749 /// vectorization) and the resume values can come from an additional bypass
750 /// block, the \p AdditionalBypass pair provides information about the bypass
751 /// block and the end value on the edge from bypass to this loop.
752 void createInductionResumeValues(
753 Loop *L, Value *VectorTripCount,
754 std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
755
756 /// Complete the loop skeleton by adding debug MDs, creating appropriate
757 /// conditional branches in the middle block, preparing the builder and
758 /// running the verifier. Take in the vector loop \p L as argument, and return
759 /// the preheader of the completed vector loop.
760 BasicBlock *completeLoopSkeleton(Loop *L, MDNode *OrigLoopID);
761
762 /// Add additional metadata to \p To that was not present on \p Orig.
763 ///
764 /// Currently this is used to add the noalias annotations based on the
765 /// inserted memchecks. Use this for instructions that are *cloned* into the
766 /// vector loop.
767 void addNewMetadata(Instruction *To, const Instruction *Orig);
768
769 /// Add metadata from one instruction to another.
770 ///
771 /// This includes both the original MDs from \p From and additional ones (\see
772 /// addNewMetadata). Use this for *newly created* instructions in the vector
773 /// loop.
774 void addMetadata(Instruction *To, Instruction *From);
775
776 /// Similar to the previous function but it adds the metadata to a
777 /// vector of instructions.
778 void addMetadata(ArrayRef<Value *> To, Instruction *From);
779
780 /// Allow subclasses to override and print debug traces before/after vplan
781 /// execution, when trace information is requested.
782 virtual void printDebugTracesAtStart(){};
783 virtual void printDebugTracesAtEnd(){};
784
785 /// The original loop.
786 Loop *OrigLoop;
787
788 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
789 /// dynamic knowledge to simplify SCEV expressions and converts them to a
790 /// more usable form.
791 PredicatedScalarEvolution &PSE;
792
793 /// Loop Info.
794 LoopInfo *LI;
795
796 /// Dominator Tree.
797 DominatorTree *DT;
798
799 /// Alias Analysis.
800 AAResults *AA;
801
802 /// Target Library Info.
803 const TargetLibraryInfo *TLI;
804
805 /// Target Transform Info.
806 const TargetTransformInfo *TTI;
807
808 /// Assumption Cache.
809 AssumptionCache *AC;
810
811 /// Interface to emit optimization remarks.
812 OptimizationRemarkEmitter *ORE;
813
814 /// LoopVersioning. It's only set up (non-null) if memchecks were
815 /// used.
816 ///
817 /// This is currently only used to add no-alias metadata based on the
818 /// memchecks. The actually versioning is performed manually.
819 std::unique_ptr<LoopVersioning> LVer;
820
821 /// The vectorization SIMD factor to use. Each vector will have this many
822 /// vector elements.
823 ElementCount VF;
824
825 /// The vectorization unroll factor to use. Each scalar is vectorized to this
826 /// many different vector instructions.
827 unsigned UF;
828
829 /// The builder that we use
830 IRBuilder<> Builder;
831
832 // --- Vectorization state ---
833
834 /// The vector-loop preheader.
835 BasicBlock *LoopVectorPreHeader;
836
837 /// The scalar-loop preheader.
838 BasicBlock *LoopScalarPreHeader;
839
840 /// Middle Block between the vector and the scalar.
841 BasicBlock *LoopMiddleBlock;
842
843 /// The (unique) ExitBlock of the scalar loop. Note that
844 /// there can be multiple exiting edges reaching this block.
845 BasicBlock *LoopExitBlock;
846
847 /// The vector loop body.
848 BasicBlock *LoopVectorBody;
849
850 /// The scalar loop body.
851 BasicBlock *LoopScalarBody;
852
853 /// A list of all bypass blocks. The first block is the entry of the loop.
854 SmallVector<BasicBlock *, 4> LoopBypassBlocks;
855
856 /// The new Induction variable which was added to the new block.
857 PHINode *Induction = nullptr;
858
859 /// The induction variable of the old basic block.
860 PHINode *OldInduction = nullptr;
861
862 /// Maps values from the original loop to their corresponding values in the
863 /// vectorized loop. A key value can map to either vector values, scalar
864 /// values or both kinds of values, depending on whether the key was
865 /// vectorized and scalarized.
866 VectorizerValueMap VectorLoopValueMap;
867
868 /// Store instructions that were predicated.
869 SmallVector<Instruction *, 4> PredicatedInstructions;
870
871 /// Trip count of the original loop.
872 Value *TripCount = nullptr;
873
874 /// Trip count of the widened loop (TripCount - TripCount % (VF*UF))
875 Value *VectorTripCount = nullptr;
876
877 /// The legality analysis.
878 LoopVectorizationLegality *Legal;
879
880 /// The profitablity analysis.
881 LoopVectorizationCostModel *Cost;
882
883 // Record whether runtime checks are added.
884 bool AddedSafetyChecks = false;
885
886 // Holds the end values for each induction variable. We save the end values
887 // so we can later fix-up the external users of the induction variables.
888 DenseMap<PHINode *, Value *> IVEndValues;
889
890 // Vector of original scalar PHIs whose corresponding widened PHIs need to be
891 // fixed up at the end of vector code generation.
892 SmallVector<PHINode *, 8> OrigPHIsToFix;
893
894 /// BFI and PSI are used to check for profile guided size optimizations.
895 BlockFrequencyInfo *BFI;
896 ProfileSummaryInfo *PSI;
897
898 // Whether this loop should be optimized for size based on profile guided size
899 // optimizatios.
900 bool OptForSizeBasedOnProfile;
901};
902
903class InnerLoopUnroller : public InnerLoopVectorizer {
904public:
905 InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
906 LoopInfo *LI, DominatorTree *DT,
907 const TargetLibraryInfo *TLI,
908 const TargetTransformInfo *TTI, AssumptionCache *AC,
909 OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,
910 LoopVectorizationLegality *LVL,
911 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
912 ProfileSummaryInfo *PSI)
913 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
914 ElementCount::getFixed(1), UnrollFactor, LVL, CM,
915 BFI, PSI) {}
916
917private:
918 Value *getBroadcastInstrs(Value *V) override;
919 Value *getStepVector(Value *Val, int StartIdx, Value *Step,
920 Instruction::BinaryOps Opcode =
921 Instruction::BinaryOpsEnd) override;
922 Value *reverseVector(Value *Vec) override;
923};
924
925/// Encapsulate information regarding vectorization of a loop and its epilogue.
926/// This information is meant to be updated and used across two stages of
927/// epilogue vectorization.
928struct EpilogueLoopVectorizationInfo {
929 ElementCount MainLoopVF = ElementCount::getFixed(0);
930 unsigned MainLoopUF = 0;
931 ElementCount EpilogueVF = ElementCount::getFixed(0);
932 unsigned EpilogueUF = 0;
933 BasicBlock *MainLoopIterationCountCheck = nullptr;
934 BasicBlock *EpilogueIterationCountCheck = nullptr;
935 BasicBlock *SCEVSafetyCheck = nullptr;
936 BasicBlock *MemSafetyCheck = nullptr;
937 Value *TripCount = nullptr;
938 Value *VectorTripCount = nullptr;
939
940 EpilogueLoopVectorizationInfo(unsigned MVF, unsigned MUF, unsigned EVF,
941 unsigned EUF)
942 : MainLoopVF(ElementCount::getFixed(MVF)), MainLoopUF(MUF),
943 EpilogueVF(ElementCount::getFixed(EVF)), EpilogueUF(EUF) {
944 assert(EUF == 1 &&
945 "A high UF for the epilogue loop is likely not beneficial.");
946 }
947};
948
949/// An extension of the inner loop vectorizer that creates a skeleton for a
950/// vectorized loop that has its epilogue (residual) also vectorized.
951/// The idea is to run the vplan on a given loop twice, firstly to setup the
952/// skeleton and vectorize the main loop, and secondly to complete the skeleton
953/// from the first step and vectorize the epilogue. This is achieved by
954/// deriving two concrete strategy classes from this base class and invoking
955/// them in succession from the loop vectorizer planner.
956class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
957public:
958 InnerLoopAndEpilogueVectorizer(
959 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
960 DominatorTree *DT, const TargetLibraryInfo *TLI,
961 const TargetTransformInfo *TTI, AssumptionCache *AC,
962 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
963 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
964 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
965 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
966 EPI.MainLoopVF, EPI.MainLoopUF, LVL, CM, BFI, PSI),
967 EPI(EPI) {}
968
969 // Override this function to handle the more complex control flow around the
970 // three loops.
971 BasicBlock *createVectorizedLoopSkeleton() final override {
972 return createEpilogueVectorizedLoopSkeleton();
973 }
974
975 /// The interface for creating a vectorized skeleton using one of two
976 /// different strategies, each corresponding to one execution of the vplan
977 /// as described above.
978 virtual BasicBlock *createEpilogueVectorizedLoopSkeleton() = 0;
979
980 /// Holds and updates state information required to vectorize the main loop
981 /// and its epilogue in two separate passes. This setup helps us avoid
982 /// regenerating and recomputing runtime safety checks. It also helps us to
983 /// shorten the iteration-count-check path length for the cases where the
984 /// iteration count of the loop is so small that the main vector loop is
985 /// completely skipped.
986 EpilogueLoopVectorizationInfo &EPI;
987};
988
989/// A specialized derived class of inner loop vectorizer that performs
990/// vectorization of *main* loops in the process of vectorizing loops and their
991/// epilogues.
992class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
993public:
994 EpilogueVectorizerMainLoop(
995 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
996 DominatorTree *DT, const TargetLibraryInfo *TLI,
997 const TargetTransformInfo *TTI, AssumptionCache *AC,
998 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
999 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
1000 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
1001 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
1002 EPI, LVL, CM, BFI, PSI) {}
1003 /// Implements the interface for creating a vectorized skeleton using the
1004 /// *main loop* strategy (ie the first pass of vplan execution).
1005 BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;
1006
1007protected:
1008 /// Emits an iteration count bypass check once for the main loop (when \p
1009 /// ForEpilogue is false) and once for the epilogue loop (when \p
1010 /// ForEpilogue is true).
1011 BasicBlock *emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass,
1012 bool ForEpilogue);
1013 void printDebugTracesAtStart() override;
1014 void printDebugTracesAtEnd() override;
1015};
1016
1017// A specialized derived class of inner loop vectorizer that performs
1018// vectorization of *epilogue* loops in the process of vectorizing loops and
1019// their epilogues.
1020class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
1021public:
1022 EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
1023 LoopInfo *LI, DominatorTree *DT,
1024 const TargetLibraryInfo *TLI,
1025 const TargetTransformInfo *TTI, AssumptionCache *AC,
1026 OptimizationRemarkEmitter *ORE,
1027 EpilogueLoopVectorizationInfo &EPI,
1028 LoopVectorizationLegality *LVL,
1029 llvm::LoopVectorizationCostModel *CM,
1030 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
1031 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
1032 EPI, LVL, CM, BFI, PSI) {}
1033 /// Implements the interface for creating a vectorized skeleton using the
1034 /// *epilogue loop* strategy (ie the second pass of vplan execution).
1035 BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;
1036
1037protected:
1038 /// Emits an iteration count bypass check after the main vector loop has
1039 /// finished to see if there are any iterations left to execute by either
1040 /// the vector epilogue or the scalar epilogue.
1041 BasicBlock *emitMinimumVectorEpilogueIterCountCheck(Loop *L,
1042 BasicBlock *Bypass,
1043 BasicBlock *Insert);
1044 void printDebugTracesAtStart() override;
1045 void printDebugTracesAtEnd() override;
1046};
1047} // end namespace llvm
1048
1049/// Look for a meaningful debug location on the instruction or it's
1050/// operands.
1051static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {
1052 if (!I)
1053 return I;
1054
1055 DebugLoc Empty;
1056 if (I->getDebugLoc() != Empty)
1057 return I;
1058
1059 for (User::op_iterator OI = I->op_begin(), OE = I->op_end(); OI != OE; ++OI) {
1060 if (Instruction *OpInst = dyn_cast<Instruction>(*OI))
1061 if (OpInst->getDebugLoc() != Empty)
1062 return OpInst;
1063 }
1064
1065 return I;
1066}
1067
1068void InnerLoopVectorizer::setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr) {
1069 if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr)) {
1070 const DILocation *DIL = Inst->getDebugLoc();
1071 if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
1072 !isa<DbgInfoIntrinsic>(Inst)) {
1073 assert(!VF.isScalable() && "scalable vectors not yet supported.");
1074 auto NewDIL =
1075 DIL->cloneByMultiplyingDuplicationFactor(UF * VF.getKnownMinValue());
1076 if (NewDIL)
1077 B.SetCurrentDebugLocation(NewDIL.getValue());
1078 else
1079 LLVM_DEBUG(dbgs()
1080 << "Failed to create new discriminator: "
1081 << DIL->getFilename() << " Line: " << DIL->getLine());
1082 }
1083 else
1084 B.SetCurrentDebugLocation(DIL);
1085 } else
1086 B.SetCurrentDebugLocation(DebugLoc());
1087}
1088
1089/// Write a record \p DebugMsg about vectorization failure to the debug
1090/// output stream. If \p I is passed, it is an instruction that prevents
1091/// vectorization.
1092#ifndef NDEBUG
1093static void debugVectorizationFailure(const StringRef DebugMsg,
1094 Instruction *I) {
1095 dbgs() << "LV: Not vectorizing: " << DebugMsg;
1096 if (I != nullptr)
1097 dbgs() << " " << *I;
1098 else
1099 dbgs() << '.';
1100 dbgs() << '\n';
1101}
1102#endif
1103
1104/// Create an analysis remark that explains why vectorization failed
1105///
1106/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
1107/// RemarkName is the identifier for the remark. If \p I is passed it is an
1108/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
1109/// the location of the remark. \return the remark object that can be
1110/// streamed to.
1111static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
1112 StringRef RemarkName, Loop *TheLoop, Instruction *I) {
1113 Value *CodeRegion = TheLoop->getHeader();
1114 DebugLoc DL = TheLoop->getStartLoc();
1115
1116 if (I) {
1117 CodeRegion = I->getParent();
1118 // If there is no debug location attached to the instruction, revert back to
1119 // using the loop's.
1120 if (I->getDebugLoc())
1121 DL = I->getDebugLoc();
1122 }
1123
1124 OptimizationRemarkAnalysis R(PassName, RemarkName, DL, CodeRegion);
1125 R << "loop not vectorized: ";
1126 return R;
1127}
1128
1129/// Return a value for Step multiplied by VF.
1130static Value *createStepForVF(IRBuilder<> &B, Constant *Step, ElementCount VF) {
1131 assert(isa<ConstantInt>(Step) && "Expected an integer step");
1132 Constant *StepVal = ConstantInt::get(
1133 Step->getType(),
1134 cast<ConstantInt>(Step)->getSExtValue() * VF.getKnownMinValue());
1135 return VF.isScalable() ? B.CreateVScale(StepVal) : StepVal;
1136}
1137
1138namespace llvm {
1139
1140void reportVectorizationFailure(const StringRef DebugMsg,
1141 const StringRef OREMsg, const StringRef ORETag,
1142 OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I) {
1143 LLVM_DEBUG(debugVectorizationFailure(DebugMsg, I));
1144 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
1145 ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(),
1146 ORETag, TheLoop, I) << OREMsg);
1147}
1148
1149} // end namespace llvm
1150
1151#ifndef NDEBUG
1152/// \return string containing a file name and a line # for the given loop.
1153static std::string getDebugLocString(const Loop *L) {
1154 std::string Result;
1155 if (L) {
1156 raw_string_ostream OS(Result);
1157 if (const DebugLoc LoopDbgLoc = L->getStartLoc())
1158 LoopDbgLoc.print(OS);
1159 else
1160 // Just print the module name.
1161 OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
1162 OS.flush();
1163 }
1164 return Result;
1165}
1166#endif
1167
1168void InnerLoopVectorizer::addNewMetadata(Instruction *To,
1169 const Instruction *Orig) {
1170 // If the loop was versioned with memchecks, add the corresponding no-alias
1171 // metadata.
1172 if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
1173 LVer->annotateInstWithNoAlias(To, Orig);
1174}
1175
1176void InnerLoopVectorizer::addMetadata(Instruction *To,
1177 Instruction *From) {
1178 propagateMetadata(To, From);
1179 addNewMetadata(To, From);
1180}
1181
1182void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To,
1183 Instruction *From) {
1184 for (Value *V : To) {
1185 if (Instruction *I = dyn_cast<Instruction>(V))
1186 addMetadata(I, From);
1187 }
1188}
1189
1190namespace llvm {
1191
1192// Loop vectorization cost-model hints how the scalar epilogue loop should be
1193// lowered.
1194enum ScalarEpilogueLowering {
1195
1196 // The default: allowing scalar epilogues.
1197 CM_ScalarEpilogueAllowed,
1198
1199 // Vectorization with OptForSize: don't allow epilogues.
1200 CM_ScalarEpilogueNotAllowedOptSize,
1201
1202 // A special case of vectorisation with OptForSize: loops with a very small
1203 // trip count are considered for vectorization under OptForSize, thereby
1204 // making sure the cost of their loop body is dominant, free of runtime
1205 // guards and scalar iteration overheads.
1206 CM_ScalarEpilogueNotAllowedLowTripLoop,
1207
1208 // Loop hint predicate indicating an epilogue is undesired.
1209 CM_ScalarEpilogueNotNeededUsePredicate,
1210
1211 // Directive indicating we must either tail fold or not vectorize
1212 CM_ScalarEpilogueNotAllowedUsePredicate
1213};
1214
1215/// LoopVectorizationCostModel - estimates the expected speedups due to
1216/// vectorization.
1217/// In many cases vectorization is not profitable. This can happen because of
1218/// a number of reasons. In this class we mainly attempt to predict the
1219/// expected speedup/slowdowns due to the supported instruction set. We use the
1220/// TargetTransformInfo to query the different backends for the cost of
1221/// different operations.
1222class LoopVectorizationCostModel {
1223public:
1224 LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L,
1225 PredicatedScalarEvolution &PSE, LoopInfo *LI,
1226 LoopVectorizationLegality *Legal,
1227 const TargetTransformInfo &TTI,
1228 const TargetLibraryInfo *TLI, DemandedBits *DB,
1229 AssumptionCache *AC,
1230 OptimizationRemarkEmitter *ORE, const Function *F,
1231 const LoopVectorizeHints *Hints,
1232 InterleavedAccessInfo &IAI)
1233 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
1234 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
1235 Hints(Hints), InterleaveInfo(IAI) {}
1236
1237 /// \return An upper bound for the vectorization factor, or None if
1238 /// vectorization and interleaving should be avoided up front.
1239 Optional<ElementCount> computeMaxVF(ElementCount UserVF, unsigned UserIC);
1240
1241 /// \return True if runtime checks are required for vectorization, and false
1242 /// otherwise.
1243 bool runtimeChecksRequired();
1244
1245 /// \return The most profitable vectorization factor and the cost of that VF.
1246 /// This method checks every power of two up to MaxVF. If UserVF is not ZERO
1247 /// then this vectorization factor will be selected if vectorization is
1248 /// possible.
1249 VectorizationFactor selectVectorizationFactor(ElementCount MaxVF);
1250 VectorizationFactor
1251 selectEpilogueVectorizationFactor(const ElementCount MaxVF,
1252 const LoopVectorizationPlanner &LVP);
1253
1254 /// Setup cost-based decisions for user vectorization factor.
1255 void selectUserVectorizationFactor(ElementCount UserVF) {
1256 collectUniformsAndScalars(UserVF);
1257 collectInstsToScalarize(UserVF);
1258 }
1259
1260 /// \return The size (in bits) of the smallest and widest types in the code
1261 /// that needs to be vectorized. We ignore values that remain scalar such as
1262 /// 64 bit loop indices.
1263 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
1264
1265 /// \return The desired interleave count.
1266 /// If interleave count has been specified by metadata it will be returned.
1267 /// Otherwise, the interleave count is computed and returned. VF and LoopCost
1268 /// are the selected vectorization factor and the cost of the selected VF.
1269 unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost);
1270
1271 /// Memory access instruction may be vectorized in more than one way.
1272 /// Form of instruction after vectorization depends on cost.
1273 /// This function takes cost-based decisions for Load/Store instructions
1274 /// and collects them in a map. This decisions map is used for building
1275 /// the lists of loop-uniform and loop-scalar instructions.
1276 /// The calculated cost is saved with widening decision in order to
1277 /// avoid redundant calculations.
1278 void setCostBasedWideningDecision(ElementCount VF);
1279
1280 /// A struct that represents some properties of the register usage
1281 /// of a loop.
1282 struct RegisterUsage {
1283 /// Holds the number of loop invariant values that are used in the loop.
1284 /// The key is ClassID of target-provided register class.
1285 SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs;
1286 /// Holds the maximum number of concurrent live intervals in the loop.
1287 /// The key is ClassID of target-provided register class.
1288 SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
1289 };
1290
1291 /// \return Returns information about the register usages of the loop for the
1292 /// given vectorization factors.
1293 SmallVector<RegisterUsage, 8>
1294 calculateRegisterUsage(ArrayRef<ElementCount> VFs);
1295
1296 /// Collect values we want to ignore in the cost model.
1297 void collectValuesToIgnore();
1298
1299 /// Split reductions into those that happen in the loop, and those that happen
1300 /// outside. In loop reductions are collected into InLoopReductionChains.
1301 void collectInLoopReductions();
1302
1303 /// \returns The smallest bitwidth each instruction can be represented with.
1304 /// The vector equivalents of these instructions should be truncated to this
1305 /// type.
1306 const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
1307 return MinBWs;
1308 }
1309
1310 /// \returns True if it is more profitable to scalarize instruction \p I for
1311 /// vectorization factor \p VF.
1312 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
1313 assert(VF.isVector() &&
1314 "Profitable to scalarize relevant only for VF > 1.");
1315
1316 // Cost model is not run in the VPlan-native path - return conservative
1317 // result until this changes.
1318 if (EnableVPlanNativePath)
1319 return false;
1320
1321 auto Scalars = InstsToScalarize.find(VF);
1322 assert(Scalars != InstsToScalarize.end() &&
1323 "VF not yet analyzed for scalarization profitability");
1324 return Scalars->second.find(I) != Scalars->second.end();
1325 }
1326
1327 /// Returns true if \p I is known to be uniform after vectorization.
1328 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
1329 if (VF.isScalar())
1330 return true;
1331
1332 // Cost model is not run in the VPlan-native path - return conservative
1333 // result until this changes.
1334 if (EnableVPlanNativePath)
1335 return false;
1336
1337 auto UniformsPerVF = Uniforms.find(VF);
1338 assert(UniformsPerVF != Uniforms.end() &&
1339 "VF not yet analyzed for uniformity");
1340 return UniformsPerVF->second.count(I);
1341 }
1342
1343 /// Returns true if \p I is known to be scalar after vectorization.
1344 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
1345 if (VF.isScalar())
1346 return true;
1347
1348 // Cost model is not run in the VPlan-native path - return conservative
1349 // result until this changes.
1350 if (EnableVPlanNativePath)
1351 return false;
1352
1353 auto ScalarsPerVF = Scalars.find(VF);
1354 assert(ScalarsPerVF != Scalars.end() &&
1355 "Scalar values are not calculated for VF");
1356 return ScalarsPerVF->second.count(I);
1357 }
1358
1359 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
1360 /// for vectorization factor \p VF.
1361 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
1362 return VF.isVector() && MinBWs.find(I) != MinBWs.end() &&
1363 !isProfitableToScalarize(I, VF) &&
1364 !isScalarAfterVectorization(I, VF);
1365 }
1366
1367 /// Decision that was taken during cost calculation for memory instruction.
1368 enum InstWidening {
1369 CM_Unknown,
1370 CM_Widen, // For consecutive accesses with stride +1.
1371 CM_Widen_Reverse, // For consecutive accesses with stride -1.
1372 CM_Interleave,
1373 CM_GatherScatter,
1374 CM_Scalarize
1375 };
1376
1377 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1378 /// instruction \p I and vector width \p VF.
1379 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
1380 InstructionCost Cost) {
1381 assert(VF.isVector() && "Expected VF >=2");
1382 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1383 }
1384
1385 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1386 /// interleaving group \p Grp and vector width \p VF.
1387 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
1388 ElementCount VF, InstWidening W,
1389 InstructionCost Cost) {
1390 assert(VF.isVector() && "Expected VF >=2");
1391 /// Broadcast this decicion to all instructions inside the group.
1392 /// But the cost will be assigned to one instruction only.
1393 for (unsigned i = 0; i < Grp->getFactor(); ++i) {
1394 if (auto *I = Grp->getMember(i)) {
1395 if (Grp->getInsertPos() == I)
1396 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1397 else
1398 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);
1399 }
1400 }
1401 }
1402
1403 /// Return the cost model decision for the given instruction \p I and vector
1404 /// width \p VF. Return CM_Unknown if this instruction did not pass
1405 /// through the cost modeling.
1406 InstWidening getWideningDecision(Instruction *I, ElementCount VF) {
1407 assert(VF.isVector() && "Expected VF to be a vector VF");
1408 // Cost model is not run in the VPlan-native path - return conservative
1409 // result until this changes.
1410 if (EnableVPlanNativePath)
1411 return CM_GatherScatter;
1412
1413 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1414 auto Itr = WideningDecisions.find(InstOnVF);
1415 if (Itr == WideningDecisions.end())
1416 return CM_Unknown;
1417 return Itr->second.first;
1418 }
1419
1420 /// Return the vectorization cost for the given instruction \p I and vector
1421 /// width \p VF.
1422 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
1423 assert(VF.isVector() && "Expected VF >=2");
1424 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1425 assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() &&
1426 "The cost is not calculated");
1427 return WideningDecisions[InstOnVF].second;
1428 }
1429
1430 /// Return True if instruction \p I is an optimizable truncate whose operand
1431 /// is an induction variable. Such a truncate will be removed by adding a new
1432 /// induction variable with the destination type.
1433 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
1434 // If the instruction is not a truncate, return false.
1435 auto *Trunc = dyn_cast<TruncInst>(I);
1436 if (!Trunc)
1437 return false;
1438
1439 // Get the source and destination types of the truncate.
1440 Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF);
1441 Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF);
1442
1443 // If the truncate is free for the given types, return false. Replacing a
1444 // free truncate with an induction variable would add an induction variable
1445 // update instruction to each iteration of the loop. We exclude from this
1446 // check the primary induction variable since it will need an update
1447 // instruction regardless.
1448 Value *Op = Trunc->getOperand(0);
1449 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1450 return false;
1451
1452 // If the truncated value is not an induction variable, return false.
1453 return Legal->isInductionPhi(Op);
1454 }
1455
1456 /// Collects the instructions to scalarize for each predicated instruction in
1457 /// the loop.
1458 void collectInstsToScalarize(ElementCount VF);
1459
1460 /// Collect Uniform and Scalar values for the given \p VF.
1461 /// The sets depend on CM decision for Load/Store instructions
1462 /// that may be vectorized as interleave, gather-scatter or scalarized.
1463 void collectUniformsAndScalars(ElementCount VF) {
1464 // Do the analysis once.
1465 if (VF.isScalar() || Uniforms.find(VF) != Uniforms.end())
1466 return;
1467 setCostBasedWideningDecision(VF);
1468 collectLoopUniforms(VF);
1469 collectLoopScalars(VF);
1470 }
1471
1472 /// Returns true if the target machine supports masked store operation
1473 /// for the given \p DataType and kind of access to \p Ptr.
1474 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) {
1475 return Legal->isConsecutivePtr(Ptr) &&
1476 TTI.isLegalMaskedStore(DataType, Alignment);
1477 }
1478
1479 /// Returns true if the target machine supports masked load operation
1480 /// for the given \p DataType and kind of access to \p Ptr.
1481 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) {
1482 return Legal->isConsecutivePtr(Ptr) &&
1483 TTI.isLegalMaskedLoad(DataType, Alignment);
1484 }
1485
1486 /// Returns true if the target machine supports masked scatter operation
1487 /// for the given \p DataType.
1488 bool isLegalMaskedScatter(Type *DataType, Align Alignment) {
1489 return TTI.isLegalMaskedScatter(DataType, Alignment);
1490 }
1491
1492 /// Returns true if the target machine supports masked gather operation
1493 /// for the given \p DataType.
1494 bool isLegalMaskedGather(Type *DataType, Align Alignment) {
1495 return TTI.isLegalMaskedGather(DataType, Alignment);
1496 }
1497
1498 /// Returns true if the target machine can represent \p V as a masked gather
1499 /// or scatter operation.
1500 bool isLegalGatherOrScatter(Value *V) {
1501 bool LI = isa<LoadInst>(V);
1502 bool SI = isa<StoreInst>(V);
1503 if (!LI && !SI)
1504 return false;
1505 auto *Ty = getMemInstValueType(V);
1506 Align Align = getLoadStoreAlignment(V);
1507 return (LI && isLegalMaskedGather(Ty, Align)) ||
1508 (SI && isLegalMaskedScatter(Ty, Align));
1509 }
1510
1511 /// Returns true if \p I is an instruction that will be scalarized with
1512 /// predication. Such instructions include conditional stores and
1513 /// instructions that may divide by zero.
1514 /// If a non-zero VF has been calculated, we check if I will be scalarized
1515 /// predication for that VF.
1516 bool isScalarWithPredication(Instruction *I,
1517 ElementCount VF = ElementCount::getFixed(1));
1518
1519 // Returns true if \p I is an instruction that will be predicated either
1520 // through scalar predication or masked load/store or masked gather/scatter.
1521 // Superset of instructions that return true for isScalarWithPredication.
1522 bool isPredicatedInst(Instruction *I) {
1523 if (!blockNeedsPredication(I->getParent()))
1524 return false;
1525 // Loads and stores that need some form of masked operation are predicated
1526 // instructions.
1527 if (isa<LoadInst>(I) || isa<StoreInst>(I))
1528 return Legal->isMaskRequired(I);
1529 return isScalarWithPredication(I);
1530 }
1531
1532 /// Returns true if \p I is a memory instruction with consecutive memory
1533 /// access that can be widened.
1534 bool
1535 memoryInstructionCanBeWidened(Instruction *I,
1536 ElementCount VF = ElementCount::getFixed(1));
1537
1538 /// Returns true if \p I is a memory instruction in an interleaved-group
1539 /// of memory accesses that can be vectorized with wide vector loads/stores
1540 /// and shuffles.
1541 bool
1542 interleavedAccessCanBeWidened(Instruction *I,
1543 ElementCount VF = ElementCount::getFixed(1));
1544
1545 /// Check if \p Instr belongs to any interleaved access group.
1546 bool isAccessInterleaved(Instruction *Instr) {
1547 return InterleaveInfo.isInterleaved(Instr);
1548 }
1549
1550 /// Get the interleaved access group that \p Instr belongs to.
1551 const InterleaveGroup<Instruction> *
1552 getInterleavedAccessGroup(Instruction *Instr) {
1553 return InterleaveInfo.getInterleaveGroup(Instr);
1554 }
1555
1556 /// Returns true if we're required to use a scalar epilogue for at least
1557 /// the final iteration of the original loop.
1558 bool requiresScalarEpilogue() const {
1559 if (!isScalarEpilogueAllowed())
1560 return false;
1561 // If we might exit from anywhere but the latch, must run the exiting
1562 // iteration in scalar form.
1563 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch())
1564 return true;
1565 return InterleaveInfo.requiresScalarEpilogue();
1566 }
1567
1568 /// Returns true if a scalar epilogue is not allowed due to optsize or a
1569 /// loop hint annotation.
1570 bool isScalarEpilogueAllowed() const {
1571 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
1572 }
1573
1574 /// Returns true if all loop blocks should be masked to fold tail loop.
1575 bool foldTailByMasking() const { return FoldTailByMasking; }
1576
1577 bool blockNeedsPredication(BasicBlock *BB) {
1578 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1579 }
1580
1581 /// A SmallMapVector to store the InLoop reduction op chains, mapping phi
1582 /// nodes to the chain of instructions representing the reductions. Uses a
1583 /// MapVector to ensure deterministic iteration order.
1584 using ReductionChainMap =
1585 SmallMapVector<PHINode *, SmallVector<Instruction *, 4>, 4>;
1586
1587 /// Return the chain of instructions representing an inloop reduction.
1588 const ReductionChainMap &getInLoopReductionChains() const {
1589 return InLoopReductionChains;
1590 }
1591
1592 /// Returns true if the Phi is part of an inloop reduction.
1593 bool isInLoopReduction(PHINode *Phi) const {
1594 return InLoopReductionChains.count(Phi);
1595 }
1596
1597 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1598 /// with factor VF. Return the cost of the instruction, including
1599 /// scalarization overhead if it's needed.
1600 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF);
1601
1602 /// Estimate cost of a call instruction CI if it were vectorized with factor
1603 /// VF. Return the cost of the instruction, including scalarization overhead
1604 /// if it's needed. The flag NeedToScalarize shows if the call needs to be
1605 /// scalarized -
1606 /// i.e. either vector version isn't available, or is too expensive.
1607 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF,
1608 bool &NeedToScalarize);
1609
1610 /// Invalidates decisions already taken by the cost model.
1611 void invalidateCostModelingDecisions() {
1612 WideningDecisions.clear();
1613 Uniforms.clear();
1614 Scalars.clear();
1615 }
1616
1617private:
1618 unsigned NumPredStores = 0;
1619
1620 /// \return An upper bound for the vectorization factor, a power-of-2 larger
1621 /// than zero. One is returned if vectorization should best be avoided due
1622 /// to cost.
1623 ElementCount computeFeasibleMaxVF(unsigned ConstTripCount,
1624 ElementCount UserVF);
1625
1626 /// The vectorization cost is a combination of the cost itself and a boolean
1627 /// indicating whether any of the contributing operations will actually
1628 /// operate on
1629 /// vector values after type legalization in the backend. If this latter value
1630 /// is
1631 /// false, then all operations will be scalarized (i.e. no vectorization has
1632 /// actually taken place).
1633 using VectorizationCostTy = std::pair<InstructionCost, bool>;
1634
1635 /// Returns the expected execution cost. The unit of the cost does
1636 /// not matter because we use the 'cost' units to compare different
1637 /// vector widths. The cost that is returned is *not* normalized by
1638 /// the factor width.
1639 VectorizationCostTy expectedCost(ElementCount VF);
1640
1641 /// Returns the execution time cost of an instruction for a given vector
1642 /// width. Vector width of one means scalar.
1643 VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF);
1644
1645 /// The cost-computation logic from getInstructionCost which provides
1646 /// the vector type as an output parameter.
1647 InstructionCost getInstructionCost(Instruction *I, ElementCount VF,
1648 Type *&VectorTy);
1649
1650 /// Return the cost of instructions in an inloop reduction pattern, if I is
1651 /// part of that pattern.
1652 InstructionCost getReductionPatternCost(Instruction *I, ElementCount VF,
1653 Type *VectorTy,
1654 TTI::TargetCostKind CostKind);
1655
1656 /// Calculate vectorization cost of memory instruction \p I.
1657 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1658
1659 /// The cost computation for scalarized memory instruction.
1660 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1661
1662 /// The cost computation for interleaving group of memory instructions.
1663 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1664
1665 /// The cost computation for Gather/Scatter instruction.
1666 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1667
1668 /// The cost computation for widening instruction \p I with consecutive
1669 /// memory access.
1670 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1671
1672 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1673 /// Load: scalar load + broadcast.
1674 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1675 /// element)
1676 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1677
1678 /// Estimate the overhead of scalarizing an instruction. This is a
1679 /// convenience wrapper for the type-based getScalarizationOverhead API.
1680 InstructionCost getScalarizationOverhead(Instruction *I, ElementCount VF);
1681
1682 /// Returns whether the instruction is a load or store and will be a emitted
1683 /// as a vector operation.
1684 bool isConsecutiveLoadOrStore(Instruction *I);
1685
1686 /// Returns true if an artificially high cost for emulated masked memrefs
1687 /// should be used.
1688 bool useEmulatedMaskMemRefHack(Instruction *I);
1689
1690 /// Map of scalar integer values to the smallest bitwidth they can be legally
1691 /// represented as. The vector equivalents of these values should be truncated
1692 /// to this type.
1693 MapVector<Instruction *, uint64_t> MinBWs;
1694
1695 /// A type representing the costs for instructions if they were to be
1696 /// scalarized rather than vectorized. The entries are Instruction-Cost
1697 /// pairs.
1698 using ScalarCostsTy = DenseMap<Instruction *, InstructionCost>;
1699
1700 /// A set containing all BasicBlocks that are known to present after
1701 /// vectorization as a predicated block.
1702 SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;
1703
1704 /// Records whether it is allowed to have the original scalar loop execute at
1705 /// least once. This may be needed as a fallback loop in case runtime
1706 /// aliasing/dependence checks fail, or to handle the tail/remainder
1707 /// iterations when the trip count is unknown or doesn't divide by the VF,
1708 /// or as a peel-loop to handle gaps in interleave-groups.
1709 /// Under optsize and when the trip count is very small we don't allow any
1710 /// iterations to execute in the scalar loop.
1711 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
1712
1713 /// All blocks of loop are to be masked to fold tail of scalar iterations.
1714 bool FoldTailByMasking = false;
1715
1716 /// A map holding scalar costs for different vectorization factors. The
1717 /// presence of a cost for an instruction in the mapping indicates that the
1718 /// instruction will be scalarized when vectorizing with the associated
1719 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1720 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
1721
1722 /// Holds the instructions known to be uniform after vectorization.
1723 /// The data is collected per VF.
1724 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1725
1726 /// Holds the instructions known to be scalar after vectorization.
1727 /// The data is collected per VF.
1728 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1729
1730 /// Holds the instructions (address computations) that are forced to be
1731 /// scalarized.
1732 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1733
1734 /// PHINodes of the reductions that should be expanded in-loop along with
1735 /// their associated chains of reduction operations, in program order from top
1736 /// (PHI) to bottom
1737 ReductionChainMap InLoopReductionChains;
1738
1739 /// A Map of inloop reduction operations and their immediate chain operand.
1740 /// FIXME: This can be removed once reductions can be costed correctly in
1741 /// vplan. This was added to allow quick lookup to the inloop operations,
1742 /// without having to loop through InLoopReductionChains.
1743 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1744
1745 /// Returns the expected difference in cost from scalarizing the expression
1746 /// feeding a predicated instruction \p PredInst. The instructions to
1747 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1748 /// non-negative return value implies the expression will be scalarized.
1749 /// Currently, only single-use chains are considered for scalarization.
1750 int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts,
1751 ElementCount VF);
1752
1753 /// Collect the instructions that are uniform after vectorization. An
1754 /// instruction is uniform if we represent it with a single scalar value in
1755 /// the vectorized loop corresponding to each vector iteration. Examples of
1756 /// uniform instructions include pointer operands of consecutive or
1757 /// interleaved memory accesses. Note that although uniformity implies an
1758 /// instruction will be scalar, the reverse is not true. In general, a
1759 /// scalarized instruction will be represented by VF scalar values in the
1760 /// vectorized loop, each corresponding to an iteration of the original
1761 /// scalar loop.
1762 void collectLoopUniforms(ElementCount VF);
1763
1764 /// Collect the instructions that are scalar after vectorization. An
1765 /// instruction is scalar if it is known to be uniform or will be scalarized
1766 /// during vectorization. Non-uniform scalarized instructions will be
1767 /// represented by VF values in the vectorized loop, each corresponding to an
1768 /// iteration of the original scalar loop.
1769 void collectLoopScalars(ElementCount VF);
1770
1771 /// Keeps cost model vectorization decision and cost for instructions.
1772 /// Right now it is used for memory instructions only.
1773 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1774 std::pair<InstWidening, InstructionCost>>;
1775
1776 DecisionList WideningDecisions;
1777
1778 /// Returns true if \p V is expected to be vectorized and it needs to be
1779 /// extracted.
1780 bool needsExtract(Value *V, ElementCount VF) const {
1781 Instruction *I = dyn_cast<Instruction>(V);
1782 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1783 TheLoop->isLoopInvariant(I))
1784 return false;
1785
1786 // Assume we can vectorize V (and hence we need extraction) if the
1787 // scalars are not computed yet. This can happen, because it is called
1788 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1789 // the scalars are collected. That should be a safe assumption in most
1790 // cases, because we check if the operands have vectorizable types
1791 // beforehand in LoopVectorizationLegality.
1792 return Scalars.find(VF) == Scalars.end() ||
1793 !isScalarAfterVectorization(I, VF);
1794 };
1795
1796 /// Returns a range containing only operands needing to be extracted.
1797 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1798 ElementCount VF) {
1799 return SmallVector<Value *, 4>(make_filter_range(
1800 Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
1801 }
1802
1803 /// Determines if we have the infrastructure to vectorize loop \p L and its
1804 /// epilogue, assuming the main loop is vectorized by \p VF.
1805 bool isCandidateForEpilogueVectorization(const Loop &L,
1806 const ElementCount VF) const;
1807
1808 /// Returns true if epilogue vectorization is considered profitable, and
1809 /// false otherwise.
1810 /// \p VF is the vectorization factor chosen for the original loop.
1811 bool isEpilogueVectorizationProfitable(const ElementCount VF) const;
1812
1813public:
1814 /// The loop that we evaluate.
1815 Loop *TheLoop;
1816
1817 /// Predicated scalar evolution analysis.
1818 PredicatedScalarEvolution &PSE;
1819
1820 /// Loop Info analysis.
1821 LoopInfo *LI;
1822
1823 /// Vectorization legality.
1824 LoopVectorizationLegality *Legal;
1825
1826 /// Vector target information.
1827 const TargetTransformInfo &TTI;
1828
1829 /// Target Library Info.
1830 const TargetLibraryInfo *TLI;
1831
1832 /// Demanded bits analysis.
1833 DemandedBits *DB;
1834
1835 /// Assumption cache.
1836 AssumptionCache *AC;
1837
1838 /// Interface to emit optimization remarks.
1839 OptimizationRemarkEmitter *ORE;
1840
1841 const Function *TheFunction;
1842
1843 /// Loop Vectorize Hint.
1844 const LoopVectorizeHints *Hints;
1845
1846 /// The interleave access information contains groups of interleaved accesses
1847 /// with the same stride and close to each other.
1848 InterleavedAccessInfo &InterleaveInfo;
1849
1850 /// Values to ignore in the cost model.
1851 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1852
1853 /// Values to ignore in the cost model when VF > 1.
1854 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1855
1856 /// Profitable vector factors.
1857 SmallVector<VectorizationFactor, 8> ProfitableVFs;
1858};
1859
1860} // end namespace llvm
1861
1862// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1863// vectorization. The loop needs to be annotated with #pragma omp simd
1864// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1865// vector length information is not provided, vectorization is not considered
1866// explicit. Interleave hints are not allowed either. These limitations will be
1867// relaxed in the future.
1868// Please, note that we are currently forced to abuse the pragma 'clang
1869// vectorize' semantics. This pragma provides *auto-vectorization hints*
1870// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1871// provides *explicit vectorization hints* (LV can bypass legal checks and
1872// assume that vectorization is legal). However, both hints are implemented
1873// using the same metadata (llvm.loop.vectorize, processed by
1874// LoopVectorizeHints). This will be fixed in the future when the native IR
1875// representation for pragma 'omp simd' is introduced.
1876static bool isExplicitVecOuterLoop(Loop *OuterLp,
1877 OptimizationRemarkEmitter *ORE) {
1878 assert(!OuterLp->isInnermost() && "This is not an outer loop");
1879 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
1880
1881 // Only outer loops with an explicit vectorization hint are supported.
1882 // Unannotated outer loops are ignored.
1883 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1884 return false;
1885
1886 Function *Fn = OuterLp->getHeader()->getParent();
1887 if (!Hints.allowVectorization(Fn, OuterLp,
1888 true /*VectorizeOnlyWhenForced*/)) {
1889 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
1890 return false;
1891 }
1892
1893 if (Hints.getInterleave() > 1) {
1894 // TODO: Interleave support is future work.
1895 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
1896 "outer loops.\n");
1897 Hints.emitRemarkWithHints();
1898 return false;
1899 }
1900
1901 return true;
1902}
1903
1904static void collectSupportedLoops(Loop &L, LoopInfo *LI,
1905 OptimizationRemarkEmitter *ORE,
1906 SmallVectorImpl<Loop *> &V) {
1907 // Collect inner loops and outer loops without irreducible control flow. For
1908 // now, only collect outer loops that have explicit vectorization hints. If we
1909 // are stress testing the VPlan H-CFG construction, we collect the outermost
1910 // loop of every loop nest.
1911 if (L.isInnermost() || VPlanBuildStressTest ||
1912 (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
1913 LoopBlocksRPO RPOT(&L);
1914 RPOT.perform(LI);
1915 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
1916 V.push_back(&L);
1917 // TODO: Collect inner loops inside marked outer loops in case
1918 // vectorization fails for the outer loop. Do not invoke
1919 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1920 // already known to be reducible. We can use an inherited attribute for
1921 // that.
1922 return;
1923 }
1924 }
1925 for (Loop *InnerL : L)
1926 collectSupportedLoops(*InnerL, LI, ORE, V);
1927}
1928
1929namespace {
1930
1931/// The LoopVectorize Pass.
1932struct LoopVectorize : public FunctionPass {
1933 /// Pass identification, replacement for typeid
1934 static char ID;
1935
1936 LoopVectorizePass Impl;
1937
1938 explicit LoopVectorize(bool InterleaveOnlyWhenForced = false,
1939 bool VectorizeOnlyWhenForced = false)
1940 : FunctionPass(ID),
1941 Impl({InterleaveOnlyWhenForced, VectorizeOnlyWhenForced}) {
1942 initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
1943 }
1944
1945 bool runOnFunction(Function &F) override {
1946 if (skipFunction(F))
1947 return false;
1948
1949 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
1950 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1951 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
1952 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1953 auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
1954 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
1955 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
1956 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1957 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
1958 auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
1959 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
1960 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
1961 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1962
1963 std::function<const LoopAccessInfo &(Loop &)> GetLAA =
1964 [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); };
1965
1966 return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC,
1967 GetLAA, *ORE, PSI).MadeAnyChange;
1968 }
1969
1970 void getAnalysisUsage(AnalysisUsage &AU) const override {
1971 AU.addRequired<AssumptionCacheTracker>();
1972 AU.addRequired<BlockFrequencyInfoWrapperPass>();
1973 AU.addRequired<DominatorTreeWrapperPass>();
1974 AU.addRequired<LoopInfoWrapperPass>();
1975 AU.addRequired<ScalarEvolutionWrapperPass>();
1976 AU.addRequired<TargetTransformInfoWrapperPass>();
1977 AU.addRequired<AAResultsWrapperPass>();
1978 AU.addRequired<LoopAccessLegacyAnalysis>();
1979 AU.addRequired<DemandedBitsWrapperPass>();
1980 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
1981 AU.addRequired<InjectTLIMappingsLegacy>();
1982
1983 // We currently do not preserve loopinfo/dominator analyses with outer loop
1984 // vectorization. Until this is addressed, mark these analyses as preserved
1985 // only for non-VPlan-native path.
1986 // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
1987 if (!EnableVPlanNativePath) {
1988 AU.addPreserved<LoopInfoWrapperPass>();
1989 AU.addPreserved<DominatorTreeWrapperPass>();
1990 }
1991
1992 AU.addPreserved<BasicAAWrapperPass>();
1993 AU.addPreserved<GlobalsAAWrapperPass>();
1994 AU.addRequired<ProfileSummaryInfoWrapperPass>();
1995 }
1996};
1997
1998} // end anonymous namespace
1999
2000//===----------------------------------------------------------------------===//
2001// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
2002// LoopVectorizationCostModel and LoopVectorizationPlanner.
2003//===----------------------------------------------------------------------===//
2004
2005Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
2006 // We need to place the broadcast of invariant variables outside the loop,
2007 // but only if it's proven safe to do so. Else, broadcast will be inside
2008 // vector loop body.
2009 Instruction *Instr = dyn_cast<Instruction>(V);
2010 bool SafeToHoist = OrigLoop->isLoopInvariant(V) &&
2011 (!Instr ||
2012 DT->dominates(Instr->getParent(), LoopVectorPreHeader));
2013 // Place the code for broadcasting invariant variables in the new preheader.
2014 IRBuilder<>::InsertPointGuard Guard(Builder);
2015 if (SafeToHoist)
2016 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
2017
2018 // Broadcast the scalar into all locations in the vector.
2019 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
2020
2021 return Shuf;
2022}
2023
2024void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
2025 const InductionDescriptor &II, Value *Step, Value *Start,
2026 Instruction *EntryVal) {
2027 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
2028 "Expected either an induction phi-node or a truncate of it!");
2029
2030 // Construct the initial value of the vector IV in the vector loop preheader
2031 auto CurrIP = Builder.saveIP();
2032 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
2033 if (isa<TruncInst>(EntryVal)) {
2034 assert(Start->getType()->isIntegerTy() &&
2035 "Truncation requires an integer type");
2036 auto *TruncType = cast<IntegerType>(EntryVal->getType());
2037 Step = Builder.CreateTrunc(Step, TruncType);
2038 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);
2039 }
2040 Value *SplatStart = Builder.CreateVectorSplat(VF, Start);
2041 Value *SteppedStart =
2042 getStepVector(SplatStart, 0, Step, II.getInductionOpcode());
2043
2044 // We create vector phi nodes for both integer and floating-point induction
2045 // variables. Here, we determine the kind of arithmetic we will perform.
2046 Instruction::BinaryOps AddOp;
2047 Instruction::BinaryOps MulOp;
2048 if (Step->getType()->isIntegerTy()) {
2049 AddOp = Instruction::Add;
2050 MulOp = Instruction::Mul;
2051 } else {
2052 AddOp = II.getInductionOpcode();
2053 MulOp = Instruction::FMul;
2054 }
2055
2056 // Multiply the vectorization factor by the step using integer or
2057 // floating-point arithmetic as appropriate.
2058 Value *ConstVF =
2059 getSignedIntOrFpConstant(Step->getType(), VF.getKnownMinValue());
2060 Value *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, Step, ConstVF));
2061
2062 // Create a vector splat to use in the induction update.
2063 //
2064 // FIXME: If the step is non-constant, we create the vector splat with
2065 // IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
2066 // handle a constant vector splat.
2067 assert(!VF.isScalable() && "scalable vectors not yet supported.");
2068 Value *SplatVF = isa<Constant>(Mul)
2069 ? ConstantVector::getSplat(VF, cast<Constant>(Mul))
2070 : Builder.CreateVectorSplat(VF, Mul);
2071 Builder.restoreIP(CurrIP);
2072
2073 // We may need to add the step a number of times, depending on the unroll
2074 // factor. The last of those goes into the PHI.
2075 PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind",
2076 &*LoopVectorBody->getFirstInsertionPt());
2077 VecInd->setDebugLoc(EntryVal->getDebugLoc());
2078 Instruction *LastInduction = VecInd;
2079 for (unsigned Part = 0; Part < UF; ++Part) {
2080 VectorLoopValueMap.setVectorValue(EntryVal, Part, LastInduction);
2081
2082 if (isa<TruncInst>(EntryVal))
2083 addMetadata(LastInduction, EntryVal);
2084 recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, Part);
2085
2086 LastInduction = cast<Instruction>(addFastMathFlag(
2087 Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add")));
2088 LastInduction->setDebugLoc(EntryVal->getDebugLoc());
2089 }
2090
2091 // Move the last step to the end of the latch block. This ensures consistent
2092 // placement of all induction updates.
2093 auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
2094 auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator());
2095 auto *ICmp = cast<Instruction>(Br->getCondition());
2096 LastInduction->moveBefore(ICmp);
2097 LastInduction->setName("vec.ind.next");
2098
2099 VecInd->addIncoming(SteppedStart, LoopVectorPreHeader);
2100 VecInd->addIncoming(LastInduction, LoopVectorLatch);
2101}
2102
2103bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const {
2104 return Cost->isScalarAfterVectorization(I, VF) ||
2105 Cost->isProfitableToScalarize(I, VF);
2106}
2107
2108bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const {
2109 if (shouldScalarizeInstruction(IV))
2110 return true;
2111 auto isScalarInst = [&](User *U) -> bool {
2112 auto *I = cast<Instruction>(U);
2113 return (OrigLoop->contains(I) && shouldScalarizeInstruction(I));
2114 };
2115 return llvm::any_of(IV->users(), isScalarInst);
2116}
2117
2118void InnerLoopVectorizer::recordVectorLoopValueForInductionCast(
2119 const InductionDescriptor &ID, const Instruction *EntryVal,
2120 Value *VectorLoopVal, unsigned Part, unsigned Lane) {
2121 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
2122 "Expected either an induction phi-node or a truncate of it!");
2123
2124 // This induction variable is not the phi from the original loop but the
2125 // newly-created IV based on the proof that casted Phi is equal to the
2126 // uncasted Phi in the vectorized loop (under a runtime guard possibly). It
2127 // re-uses the same InductionDescriptor that original IV uses but we don't
2128 // have to do any recording in this case - that is done when original IV is
2129 // processed.
2130 if (isa<TruncInst>(EntryVal))
2131 return;
2132
2133 const SmallVectorImpl<Instruction *> &Casts = ID.getCastInsts();
2134 if (Casts.empty())
2135 return;
2136 // Only the first Cast instruction in the Casts vector is of interest.
2137 // The rest of the Casts (if exist) have no uses outside the
2138 // induction update chain itself.
2139 Instruction *CastInst = *Casts.begin();
2140 if (Lane < UINT_MAX)
2141 VectorLoopValueMap.setScalarValue(CastInst, {Part, Lane}, VectorLoopVal);
2142 else
2143 VectorLoopValueMap.setVectorValue(CastInst, Part, VectorLoopVal);
2144}
2145
2146void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
2147 TruncInst *Trunc) {
2148 assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&
2149 "Primary induction variable must have an integer type");
2150
2151 auto II = Legal->getInductionVars().find(IV);
2152 assert(II != Legal->getInductionVars().end() && "IV is not an induction");
2153
2154 auto ID = II->second;
2155 assert(IV->getType() == ID.getStartValue()->getType() && "Types must match");
2156
2157 // The value from the original loop to which we are mapping the new induction
2158 // variable.
2159 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
2160
2161 auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
2162
2163 // Generate code for the induction step. Note that induction steps are
2164 // required to be loop-invariant
2165 auto CreateStepValue = [&](const SCEV *Step) -> Value * {
2166 assert(PSE.getSE()->isLoopInvariant(Step, OrigLoop) &&
2167 "Induction step should be loop invariant");
2168 if (PSE.getSE()->isSCEVable(IV->getType())) {
2169 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
2170 return Exp.expandCodeFor(Step, Step->getType(),
2171 LoopVectorPreHeader->getTerminator());
2172 }
2173 return cast<SCEVUnknown>(Step)->getValue();
2174 };
2175
2176 // The scalar value to broadcast. This is derived from the canonical
2177 // induction variable. If a truncation type is given, truncate the canonical
2178 // induction variable and step. Otherwise, derive these values from the
2179 // induction descriptor.
2180 auto CreateScalarIV = [&](Value *&Step) -> Value * {
2181 Value *ScalarIV = Induction;
2182 if (IV != OldInduction) {
2183 ScalarIV = IV->getType()->isIntegerTy()
2184 ? Builder.CreateSExtOrTrunc(Induction, IV->getType())
2185 : Builder.CreateCast(Instruction::SIToFP, Induction,
2186 IV->getType());
2187 ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID);
2188 ScalarIV->setName("offset.idx");
2189 }
2190 if (Trunc) {
2191 auto *TruncType = cast<IntegerType>(Trunc->getType());
2192 assert(Step->getType()->isIntegerTy() &&
2193 "Truncation requires an integer step");
2194 ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType);
2195 Step = Builder.CreateTrunc(Step, TruncType);
2196 }
2197 return ScalarIV;
2198 };
2199
2200 // Create the vector values from the scalar IV, in the absence of creating a
2201 // vector IV.
2202 auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) {
2203 Value *Broadcasted = getBroadcastInstrs(ScalarIV);
2204 for (unsigned Part = 0; Part < UF; ++Part) {
2205 assert(!VF.isScalable() && "scalable vectors not yet supported.");
2206 Value *EntryPart =
2207 getStepVector(Broadcasted, VF.getKnownMinValue() * Part, Step,
2208 ID.getInductionOpcode());
2209 VectorLoopValueMap.setVectorValue(EntryVal, Part, EntryPart);
2210 if (Trunc)
2211 addMetadata(EntryPart, Trunc);
2212 recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, Part);
2213 }
2214 };
2215
2216 // Now do the actual transformations, and start with creating the step value.
2217 Value *Step = CreateStepValue(ID.getStep());
2218 if (VF.isZero() || VF.isScalar()) {
2219 Value *ScalarIV = CreateScalarIV(Step);
2220 CreateSplatIV(ScalarIV, Step);
2221 return;
2222 }
2223
2224 // Determine if we want a scalar version of the induction variable. This is
2225 // true if the induction variable itself is not widened, or if it has at
2226 // least one user in the loop that is not widened.
2227 auto NeedsScalarIV = needsScalarInduction(EntryVal);
2228 if (!NeedsScalarIV) {
2229 createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal);
2230 return;
2231 }
2232
2233 // Try to create a new independent vector induction variable. If we can't
2234 // create the phi node, we will splat the scalar induction variable in each
2235 // loop iteration.
2236 if (!shouldScalarizeInstruction(EntryVal)) {
2237 createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal);
2238 Value *ScalarIV = CreateScalarIV(Step);
2239 // Create scalar steps that can be used by instructions we will later
2240 // scalarize. Note that the addition of the scalar steps will not increase
2241 // the number of instructions in the loop in the common case prior to
2242 // InstCombine. We will be trading one vector extract for each scalar step.
2243 buildScalarSteps(ScalarIV, Step, EntryVal, ID);
2244 return;
2245 }
2246
2247 // All IV users are scalar instructions, so only emit a scalar IV, not a
2248 // vectorised IV. Except when we tail-fold, then the splat IV feeds the
2249 // predicate used by the masked loads/stores.
2250 Value *ScalarIV = CreateScalarIV(Step);
2251 if (!Cost->isScalarEpilogueAllowed())
2252 CreateSplatIV(ScalarIV, Step);
2253 buildScalarSteps(ScalarIV, Step, EntryVal, ID);
2254}
2255
2256Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,
2257 Instruction::BinaryOps BinOp) {
2258 // Create and check the types.
2259 auto *ValVTy = cast<FixedVectorType>(Val->getType());
2260 int VLen = ValVTy->getNumElements();
2261
2262 Type *STy = Val->getType()->getScalarType();
2263 assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
2264 "Induction Step must be an integer or FP");
2265 assert(Step->getType() == STy && "Step has wrong type");
2266
2267 SmallVector<Constant *, 8> Indices;
2268
2269 if (STy->isIntegerTy()) {
2270 // Create a vector of consecutive numbers from zero to VF.
2271 for (int i = 0; i < VLen; ++i)
2272 Indices.push_back(ConstantInt::get(STy, StartIdx + i));
2273
2274 // Add the consecutive indices to the vector value.
2275 Constant *Cv = ConstantVector::get(Indices);
2276 assert(Cv->getType() == Val->getType() && "Invalid consecutive vec");
2277 Step = Builder.CreateVectorSplat(VLen, Step);
2278 assert(Step->getType() == Val->getType() && "Invalid step vec");
2279 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
2280 // which can be found from the original scalar operations.
2281 Step = Builder.CreateMul(Cv, Step);
2282 return Builder.CreateAdd(Val, Step, "induction");
2283 }
2284
2285 // Floating point induction.
2286 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
2287 "Binary Opcode should be specified for FP induction");
2288 // Create a vector of consecutive numbers from zero to VF.
2289 for (int i = 0; i < VLen; ++i)
2290 Indices.push_back(ConstantFP::get(STy, (double)(StartIdx + i)));
2291
2292 // Add the consecutive indices to the vector value.
2293 Constant *Cv = ConstantVector::get(Indices);
2294
2295 Step = Builder.CreateVectorSplat(VLen, Step);
2296
2297 // Floating point operations had to be 'fast' to enable the induction.
2298 FastMathFlags Flags;
2299 Flags.setFast();
2300
2301 Value *MulOp = Builder.CreateFMul(Cv, Step);
2302 if (isa<Instruction>(MulOp))
2303 // Have to check, MulOp may be a constant
2304 cast<Instruction>(MulOp)->setFastMathFlags(Flags);
2305
2306 Value *BOp = Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
2307 if (isa<Instruction>(BOp))
2308 cast<Instruction>(BOp)->setFastMathFlags(Flags);
2309 return BOp;
2310}
2311
2312void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
2313 Instruction *EntryVal,
2314 const InductionDescriptor &ID) {
2315 // We shouldn't have to build scalar steps if we aren't vectorizing.
2316 assert(VF.isVector() && "VF should be greater than one");
2317 // Get the value type and ensure it and the step have the same integer type.
2318 Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
2319 assert(ScalarIVTy == Step->getType() &&
2320 "Val and Step should have the same type");
2321
2322 // We build scalar steps for both integer and floating-point induction
2323 // variables. Here, we determine the kind of arithmetic we will perform.
2324 Instruction::BinaryOps AddOp;
2325 Instruction::BinaryOps MulOp;
2326 if (ScalarIVTy->isIntegerTy()) {
2327 AddOp = Instruction::Add;
2328 MulOp = Instruction::Mul;
2329 } else {
2330 AddOp = ID.getInductionOpcode();
2331 MulOp = Instruction::FMul;
2332 }
2333
2334 // Determine the number of scalars we need to generate for each unroll
2335 // iteration. If EntryVal is uniform, we only need to generate the first
2336 // lane. Otherwise, we generate all VF values.
2337 unsigned Lanes =
2338 Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF)
2339 ? 1
2340 : VF.getKnownMinValue();
2341 assert((!VF.isScalable() || Lanes == 1) &&
2342 "Should never scalarize a scalable vector");
2343 // Compute the scalar steps and save the results in VectorLoopValueMap.
2344 for (unsigned Part = 0; Part < UF; ++Part) {
2345 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
2346 auto *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
2347 ScalarIVTy->getScalarSizeInBits());
2348 Value *StartIdx =
2349 createStepForVF(Builder, ConstantInt::get(IntStepTy, Part), VF);
2350 if (ScalarIVTy->isFloatingPointTy())
2351 StartIdx = Builder.CreateSIToFP(StartIdx, ScalarIVTy);
2352 StartIdx = addFastMathFlag(Builder.CreateBinOp(
2353 AddOp, StartIdx, getSignedIntOrFpConstant(ScalarIVTy, Lane)));
2354 // The step returned by `createStepForVF` is a runtime-evaluated value
2355 // when VF is scalable. Otherwise, it should be folded into a Constant.
2356 assert((VF.isScalable() || isa<Constant>(StartIdx)) &&
2357 "Expected StartIdx to be folded to a constant when VF is not "
2358 "scalable");
2359 auto *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, StartIdx, Step));
2360 auto *Add = addFastMathFlag(Builder.CreateBinOp(AddOp, ScalarIV, Mul));
2361 VectorLoopValueMap.setScalarValue(EntryVal, {Part, Lane}, Add);
2362 recordVectorLoopValueForInductionCast(ID, EntryVal, Add, Part, Lane);
2363 }
2364 }
2365}
2366
2367Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) {
2368 assert(V != Induction && "The new induction variable should not be used.");
2369 assert(!V->getType()->isVectorTy() && "Can't widen a vector");
2370 assert(!V->getType()->isVoidTy() && "Type does not produce a value");
2371
2372 // If we have a stride that is replaced by one, do it here. Defer this for
2373 // the VPlan-native path until we start running Legal checks in that path.
2374 if (!EnableVPlanNativePath && Legal->hasStride(V))
2375 V = ConstantInt::get(V->getType(), 1);
2376
2377 // If we have a vector mapped to this value, return it.
2378 if (VectorLoopValueMap.hasVectorValue(V, Part))
2379 return VectorLoopValueMap.getVectorValue(V, Part);
2380
2381 // If the value has not been vectorized, check if it has been scalarized
2382 // instead. If it has been scalarized, and we actually need the value in
2383 // vector form, we will construct the vector values on demand.
2384 if (VectorLoopValueMap.hasAnyScalarValue(V)) {
2385 Value *ScalarValue = VectorLoopValueMap.getScalarValue(V, {Part, 0});
2386
2387 // If we've scalarized a value, that value should be an instruction.
2388 auto *I = cast<Instruction>(V);
2389
2390 // If we aren't vectorizing, we can just copy the scalar map values over to
2391 // the vector map.
2392 if (VF.isScalar()) {
2393 VectorLoopValueMap.setVectorValue(V, Part, ScalarValue);
2394 return ScalarValue;
2395 }
2396
2397 // Get the last scalar instruction we generated for V and Part. If the value
2398 // is known to be uniform after vectorization, this corresponds to lane zero
2399 // of the Part unroll iteration. Otherwise, the last instruction is the one
2400 // we created for the last vector lane of the Part unroll iteration.
2401 unsigned LastLane = Cost->isUniformAfterVectorization(I, VF)
2402 ? 0
2403 : VF.getKnownMinValue() - 1;
2404 assert((!VF.isScalable() || LastLane == 0) &&
2405 "Scalable vectorization can't lead to any scalarized values.");
2406 auto *LastInst = cast<Instruction>(
2407 VectorLoopValueMap.getScalarValue(V, {Part, LastLane}));
2408
2409 // Set the insert point after the last scalarized instruction. This ensures
2410 // the insertelement sequence will directly follow the scalar definitions.
2411 auto OldIP = Builder.saveIP();
2412 auto NewIP = std::next(BasicBlock::iterator(LastInst));
2413 Builder.SetInsertPoint(&*NewIP);
2414
2415 // However, if we are vectorizing, we need to construct the vector values.
2416 // If the value is known to be uniform after vectorization, we can just
2417 // broadcast the scalar value corresponding to lane zero for each unroll
2418 // iteration. Otherwise, we construct the vector values using insertelement
2419 // instructions. Since the resulting vectors are stored in
2420 // VectorLoopValueMap, we will only generate the insertelements once.
2421 Value *VectorValue = nullptr;
2422 if (Cost->isUniformAfterVectorization(I, VF)) {
2423 VectorValue = getBroadcastInstrs(ScalarValue);
2424 VectorLoopValueMap.setVectorValue(V, Part, VectorValue);
2425 } else {
2426 // Initialize packing with insertelements to start from poison.
2427 assert(!VF.isScalable() && "VF is assumed to be non scalable.");
2428 Value *Poison = PoisonValue::get(VectorType::get(V->getType(), VF));
2429 VectorLoopValueMap.setVectorValue(V, Part, Poison);
2430 for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
2431 packScalarIntoVectorValue(V, {Part, Lane});
2432 VectorValue = VectorLoopValueMap.getVectorValue(V, Part);
2433 }
2434 Builder.restoreIP(OldIP);
2435 return VectorValue;
2436 }
2437
2438 // If this scalar is unknown, assume that it is a constant or that it is
2439 // loop invariant. Broadcast V and save the value for future uses.
2440 Value *B = getBroadcastInstrs(V);
2441 VectorLoopValueMap.setVectorValue(V, Part, B);
2442 return B;
2443}
2444
2445Value *
2446InnerLoopVectorizer::getOrCreateScalarValue(Value *V,
2447 const VPIteration &Instance) {
2448 // If the value is not an instruction contained in the loop, it should
2449 // already be scalar.
2450 if (OrigLoop->isLoopInvariant(V))
2451 return V;
2452
2453 assert(Instance.Lane > 0
2454 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF)
2455 : true && "Uniform values only have lane zero");
2456
2457 // If the value from the original loop has not been vectorized, it is
2458 // represented by UF x VF scalar values in the new loop. Return the requested
2459 // scalar value.
2460 if (VectorLoopValueMap.hasScalarValue(V, Instance))
2461 return VectorLoopValueMap.getScalarValue(V, Instance);
2462
2463 // If the value has not been scalarized, get its entry in VectorLoopValueMap
2464 // for the given unroll part. If this entry is not a vector type (i.e., the
2465 // vectorization factor is one), there is no need to generate an
2466 // extractelement instruction.
2467 auto *U = getOrCreateVectorValue(V, Instance.Part);
2468 if (!U->getType()->isVectorTy()) {
2469 assert(VF.isScalar() && "Value not scalarized has non-vector type");
2470 return U;
2471 }
2472
2473 // Otherwise, the value from the original loop has been vectorized and is
2474 // represented by UF vector values. Extract and return the requested scalar
2475 // value from the appropriate vector lane.
2476 return Builder.CreateExtractElement(U, Builder.getInt32(Instance.Lane));
2477}
2478
2479void InnerLoopVectorizer::packScalarIntoVectorValue(
2480 Value *V, const VPIteration &Instance) {
2481 assert(V != Induction && "The new induction variable should not be used.");
2482 assert(!V->getType()->isVectorTy() && "Can't pack a vector");
2483 assert(!V->getType()->isVoidTy() && "Type does not produce a value");
2484
2485 Value *ScalarInst = VectorLoopValueMap.getScalarValue(V, Instance);
2486 Value *VectorValue = VectorLoopValueMap.getVectorValue(V, Instance.Part);
2487 VectorValue = Builder.CreateInsertElement(VectorValue, ScalarInst,
2488 Builder.getInt32(Instance.Lane));
2489 VectorLoopValueMap.resetVectorValue(V, Instance.Part, VectorValue);
2490}
2491
2492Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
2493 assert(Vec->getType()->isVectorTy() && "Invalid type");
2494 assert(!VF.isScalable() && "Cannot reverse scalable vectors");
2495 SmallVector<int, 8> ShuffleMask;
2496 for (unsigned i = 0; i < VF.getKnownMinValue(); ++i)
2497 ShuffleMask.push_back(VF.getKnownMinValue() - i - 1);
2498
2499 return Builder.CreateShuffleVector(Vec, ShuffleMask, "reverse");
2500}
2501
2502// Return whether we allow using masked interleave-groups (for dealing with
2503// strided loads/stores that reside in predicated blocks, or for dealing
2504// with gaps).
2505static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
2506 // If an override option has been passed in for interleaved accesses, use it.
2507 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
2508 return EnableMaskedInterleavedMemAccesses;
2509
2510 return TTI.enableMaskedInterleavedAccessVectorization();
2511}
2512
2513// Try to vectorize the interleave group that \p Instr belongs to.
2514//
2515// E.g. Translate following interleaved load group (factor = 3):
2516// for (i = 0; i < N; i+=3) {
2517// R = Pic[i]; // Member of index 0
2518// G = Pic[i+1]; // Member of index 1
2519// B = Pic[i+2]; // Member of index 2
2520// ... // do something to R, G, B
2521// }
2522// To:
2523// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B
2524// %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9> ; R elements
2525// %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10> ; G elements
2526// %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11> ; B elements
2527//
2528// Or translate following interleaved store group (factor = 3):
2529// for (i = 0; i < N; i+=3) {
2530// ... do something to R, G, B
2531// Pic[i] = R; // Member of index 0
2532// Pic[i+1] = G; // Member of index 1
2533// Pic[i+2] = B; // Member of index 2
2534// }
2535// To:
2536// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
2537// %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
2538// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
2539// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
2540// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
2541void InnerLoopVectorizer::vectorizeInterleaveGroup(
2542 const InterleaveGroup<Instruction> *Group, ArrayRef<VPValue *> VPDefs,
2543 VPTransformState &State, VPValue *Addr, ArrayRef<VPValue *> StoredValues,
2544 VPValue *BlockInMask) {
2545 Instruction *Instr = Group->getInsertPos();
2546 const DataLayout &DL = Instr->getModule()->getDataLayout();
2547
2548 // Prepare for the vector type of the interleaved load/store.
2549 Type *ScalarTy = getMemInstValueType(Instr);
2550 unsigned InterleaveFactor = Group->getFactor();
2551 assert(!VF.isScalable() && "scalable vectors not yet supported.");
2552 auto *VecTy = VectorType::get(ScalarTy, VF * InterleaveFactor);
2553
2554 // Prepare for the new pointers.
2555 SmallVector<Value *, 2> AddrParts;
2556 unsigned Index = Group->getIndex(Instr);
2557
2558 // TODO: extend the masked interleaved-group support to reversed access.
2559 assert((!BlockInMask || !Group->isReverse()) &&
2560 "Reversed masked interleave-group not supported.");
2561
2562 // If the group is reverse, adjust the index to refer to the last vector lane
2563 // instead of the first. We adjust the index from the first vector lane,
2564 // rather than directly getting the pointer for lane VF - 1, because the
2565 // pointer operand of the interleaved access is supposed to be uniform. For
2566 // uniform instructions, we're only required to generate a value for the
2567 // first vector lane in each unroll iteration.
2568 assert(!VF.isScalable() &&
2569 "scalable vector reverse operation is not implemented");
2570 if (Group->isReverse())
2571 Index += (VF.getKnownMinValue() - 1) * Group->getFactor();
2572
2573 for (unsigned Part = 0; Part < UF; Part++) {
2574 Value *AddrPart = State.get(Addr, {Part, 0});
2575 setDebugLocFromInst(Builder, AddrPart);
2576
2577 // Notice current instruction could be any index. Need to adjust the address
2578 // to the member of index 0.
2579 //
2580 // E.g. a = A[i+1]; // Member of index 1 (Current instruction)
2581 // b = A[i]; // Member of index 0
2582 // Current pointer is pointed to A[i+1], adjust it to A[i].
2583 //
2584 // E.g. A[i+1] = a; // Member of index 1
2585 // A[i] = b; // Member of index 0
2586 // A[i+2] = c; // Member of index 2 (Current instruction)
2587 // Current pointer is pointed to A[i+2], adjust it to A[i].
2588
2589 bool InBounds = false;
2590 if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts()))
2591 InBounds = gep->isInBounds();
2592 AddrPart = Builder.CreateGEP(ScalarTy, AddrPart, Builder.getInt32(-Index));
2593 cast<GetElementPtrInst>(AddrPart)->setIsInBounds(InBounds);
2594
2595 // Cast to the vector pointer type.
2596 unsigned AddressSpace = AddrPart->getType()->getPointerAddressSpace();
2597 Type *PtrTy = VecTy->getPointerTo(AddressSpace);
2598 AddrParts.push_back(Builder.CreateBitCast(AddrPart, PtrTy));
2599 }
2600
2601 setDebugLocFromInst(Builder, Instr);
2602 Value *PoisonVec = PoisonValue::get(VecTy);
2603
2604 Value *MaskForGaps = nullptr;
2605 if (Group->requiresScalarEpilogue() && !Cost->isScalarEpilogueAllowed()) {
2606 assert(!VF.isScalable() && "scalable vectors not yet supported.");
2607 MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group);
2608 assert(MaskForGaps && "Mask for Gaps is required but it is null");
2609 }
2610
2611 // Vectorize the interleaved load group.
2612 if (isa<LoadInst>(Instr)) {
2613 // For each unroll part, create a wide load for the group.
2614 SmallVector<Value *, 2> NewLoads;
2615 for (unsigned Part = 0; Part < UF; Part++) {
2616 Instruction *NewLoad;
2617 if (BlockInMask || MaskForGaps) {
2618 assert(useMaskedInterleavedAccesses(*TTI) &&
2619 "masked interleaved groups are not allowed.");
2620 Value *GroupMask = MaskForGaps;
2621 if (BlockInMask) {
2622 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2623 assert(!VF.isScalable() && "scalable vectors not yet supported.");
2624 Value *ShuffledMask = Builder.CreateShuffleVector(
2625 BlockInMaskPart,
2626 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2627 "interleaved.mask");
2628 GroupMask = MaskForGaps
2629 ? Builder.CreateBinOp(Instruction::And, ShuffledMask,
2630 MaskForGaps)
2631 : ShuffledMask;
2632 }
2633 NewLoad =
2634 Builder.CreateMaskedLoad(AddrParts[Part], Group->getAlign(),
2635 GroupMask, PoisonVec, "wide.masked.vec");
2636 }
2637 else
2638 NewLoad = Builder.CreateAlignedLoad(VecTy, AddrParts[Part],
2639 Group->getAlign(), "wide.vec");
2640 Group->addMetadata(NewLoad);
2641 NewLoads.push_back(NewLoad);
2642 }
2643
2644 // For each member in the group, shuffle out the appropriate data from the
2645 // wide loads.
2646 unsigned J = 0;
2647 for (unsigned I = 0; I < InterleaveFactor; ++I) {
2648 Instruction *Member = Group->getMember(I);
2649
2650 // Skip the gaps in the group.
2651 if (!Member)
2652 continue;
2653
2654 assert(!VF.isScalable() && "scalable vectors not yet supported.");
2655 auto StrideMask =
2656 createStrideMask(I, InterleaveFactor, VF.getKnownMinValue());
2657 for (unsigned Part = 0; Part < UF; Part++) {
2658 Value *StridedVec = Builder.CreateShuffleVector(
2659 NewLoads[Part], StrideMask, "strided.vec");
2660
2661 // If this member has different type, cast the result type.
2662 if (Member->getType() != ScalarTy) {
2663 assert(!VF.isScalable() && "VF is assumed to be non scalable.");
2664 VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
2665 StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
2666 }
2667
2668 if (Group->isReverse())
2669 StridedVec = reverseVector(StridedVec);
2670
2671 State.set(VPDefs[J], Member, StridedVec, Part);
2672 }
2673 ++J;
2674 }
2675 return;
2676 }
2677
2678 // The sub vector type for current instruction.
2679 assert(!VF.isScalable() && "VF is assumed to be non scalable.");
2680 auto *SubVT = VectorType::get(ScalarTy, VF);
2681
2682 // Vectorize the interleaved store group.
2683 for (unsigned Part = 0; Part < UF; Part++) {
2684 // Collect the stored vector from each member.
2685 SmallVector<Value *, 4> StoredVecs;
2686 for (unsigned i = 0; i < InterleaveFactor; i++) {
2687 // Interleaved store group doesn't allow a gap, so each index has a member
2688 assert(Group->getMember(i) && "Fail to get a member from an interleaved store group");
2689
2690 Value *StoredVec = State.get(StoredValues[i], Part);
2691
2692 if (Group->isReverse())
2693 StoredVec = reverseVector(StoredVec);
2694
2695 // If this member has different type, cast it to a unified type.
2696
2697 if (StoredVec->getType() != SubVT)
2698 StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL);
2699
2700 StoredVecs.push_back(StoredVec);
2701 }
2702
2703 // Concatenate all vectors into a wide vector.
2704 Value *WideVec = concatenateVectors(Builder, StoredVecs);
2705
2706 // Interleave the elements in the wide vector.
2707 assert(!VF.isScalable() && "scalable vectors not yet supported.");
2708 Value *IVec = Builder.CreateShuffleVector(
2709 WideVec, createInterleaveMask(VF.getKnownMinValue(), InterleaveFactor),
2710 "interleaved.vec");
2711
2712 Instruction *NewStoreInstr;
2713 if (BlockInMask) {
2714 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2715 Value *ShuffledMask = Builder.CreateShuffleVector(
2716 BlockInMaskPart,
2717 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2718 "interleaved.mask");
2719 NewStoreInstr = Builder.CreateMaskedStore(
2720 IVec, AddrParts[Part], Group->getAlign(), ShuffledMask);
2721 }
2722 else
2723 NewStoreInstr =
2724 Builder.CreateAlignedStore(IVec, AddrParts[Part], Group->getAlign());
2725
2726 Group->addMetadata(NewStoreInstr);
2727 }
2728}
2729
2730void InnerLoopVectorizer::vectorizeMemoryInstruction(
2731 Instruction *Instr, VPTransformState &State, VPValue *Def, VPValue *Addr,
2732 VPValue *StoredValue, VPValue *BlockInMask) {
2733 // Attempt to issue a wide load.
2734 LoadInst *LI = dyn_cast<LoadInst>(Instr);
2735 StoreInst *SI = dyn_cast<StoreInst>(Instr);
2736
2737 assert((LI || SI) && "Invalid Load/Store instruction");
2738 assert((!SI || StoredValue) && "No stored value provided for widened store");
2739 assert((!LI || !StoredValue) && "Stored value provided for widened load");
2740
2741 LoopVectorizationCostModel::InstWidening Decision =
2742 Cost->getWideningDecision(Instr, VF);
2743 assert((Decision == LoopVectorizationCostModel::CM_Widen ||
2744 Decision == LoopVectorizationCostModel::CM_Widen_Reverse ||
2745 Decision == LoopVectorizationCostModel::CM_GatherScatter) &&
2746 "CM decision is not to widen the memory instruction");
2747
2748 Type *ScalarDataTy = getMemInstValueType(Instr);
2749
2750 auto *DataTy = VectorType::get(ScalarDataTy, VF);
2751 const Align Alignment = getLoadStoreAlignment(Instr);
2752
2753 // Determine if the pointer operand of the access is either consecutive or
2754 // reverse consecutive.
2755 bool Reverse = (Decision == LoopVectorizationCostModel::CM_Widen_Reverse);
2756 bool ConsecutiveStride =
2757 Reverse || (Decision == LoopVectorizationCostModel::CM_Widen);
2758 bool CreateGatherScatter =
2759 (Decision == LoopVectorizationCostModel::CM_GatherScatter);
2760
2761 // Either Ptr feeds a vector load/store, or a vector GEP should feed a vector
2762 // gather/scatter. Otherwise Decision should have been to Scalarize.
2763 assert((ConsecutiveStride || CreateGatherScatter) &&
2764 "The instruction should be scalarized");
2765 (void)ConsecutiveStride;
2766
2767 VectorParts BlockInMaskParts(UF);
2768 bool isMaskRequired = BlockInMask;
2769 if (isMaskRequired)
2770 for (unsigned Part = 0; Part < UF; ++Part)
2771 BlockInMaskParts[Part] = State.get(BlockInMask, Part);
2772
2773 const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * {
2774 // Calculate the pointer for the specific unroll-part.
2775 GetElementPtrInst *PartPtr = nullptr;
2776
2777 bool InBounds = false;
2778 if (auto *gep = dyn_cast<GetElementPtrInst>(Ptr->stripPointerCasts()))
2779 InBounds = gep->isInBounds();
2780
2781 if (Reverse) {
2782 assert(!VF.isScalable() &&
2783 "Reversing vectors is not yet supported for scalable vectors.");
2784
2785 // If the address is consecutive but reversed, then the
2786 // wide store needs to start at the last vector element.
2787 PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
2788 ScalarDataTy, Ptr, Builder.getInt32(-Part * VF.getKnownMinValue())));
2789 PartPtr->setIsInBounds(InBounds);
2790 PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
2791 ScalarDataTy, PartPtr, Builder.getInt32(1 - VF.getKnownMinValue())));
2792 PartPtr->setIsInBounds(InBounds);
2793 if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
2794 BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]);
2795 } else {
2796 Value *Increment = createStepForVF(Builder, Builder.getInt32(Part), VF);
2797 PartPtr = cast<GetElementPtrInst>(
2798 Builder.CreateGEP(ScalarDataTy, Ptr, Increment));
2799 PartPtr->setIsInBounds(InBounds);
2800 }
2801
2802 unsigned AddressSpace = Ptr->getType()->getPointerAddressSpace();
2803 return Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));
2804 };
2805
2806 // Handle Stores:
2807 if (SI) {
2808 setDebugLocFromInst(Builder, SI);
2809
2810 for (unsigned Part = 0; Part < UF; ++Part) {
2811 Instruction *NewSI = nullptr;
2812 Value *StoredVal = State.get(StoredValue, Part);
2813 if (CreateGatherScatter) {
2814 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
2815 Value *VectorGep = State.get(Addr, Part);
2816 NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
2817 MaskPart);
2818 } else {
2819 if (Reverse) {
2820 // If we store to reverse consecutive memory locations, then we need
2821 // to reverse the order of elements in the stored value.
2822 StoredVal = reverseVector(StoredVal);
2823 // We don't want to update the value in the map as it might be used in
2824 // another expression. So don't call resetVectorValue(StoredVal).
2825 }
2826 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, {0, 0}));
2827 if (isMaskRequired)
2828 NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
2829 BlockInMaskParts[Part]);
2830 else
2831 NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
2832 }
2833 addMetadata(NewSI, SI);
2834 }
2835 return;
2836 }
2837
2838 // Handle loads.
2839 assert(LI && "Must have a load instruction");
2840 setDebugLocFromInst(Builder, LI);
2841 for (unsigned Part = 0; Part < UF; ++Part) {
2842 Value *NewLI;
2843 if (CreateGatherScatter) {
2844 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
2845 Value *VectorGep = State.get(Addr, Part);
2846 NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart,
2847 nullptr, "wide.masked.gather");
2848 addMetadata(NewLI, LI);
2849 } else {
2850 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, {0, 0}));
2851 if (isMaskRequired)
2852 NewLI = Builder.CreateMaskedLoad(
2853 VecPtr, Alignment, BlockInMaskParts[Part], PoisonValue::get(DataTy),
2854 "wide.masked.load");
2855 else
2856 NewLI =
2857 Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
2858
2859 // Add metadata to the load, but setVectorValue to the reverse shuffle.
2860 addMetadata(NewLI, LI);
2861 if (Reverse)
2862 NewLI = reverseVector(NewLI);
2863 }
2864
2865 State.set(Def, Instr, NewLI, Part);
2866 }
2867}
2868
2869void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPUser &User,
2870 const VPIteration &Instance,
2871 bool IfPredicateInstr,
2872 VPTransformState &State) {
2873 assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
2874
2875 // llvm.experimental.noalias.scope.decl intrinsics must only be duplicated for
2876 // the first lane and part.
2877 if (isa<NoAliasScopeDeclInst>(Instr))
2878 if (Instance.Lane != 0 || Instance.Part != 0)
2879 return;
2880
2881 setDebugLocFromInst(Builder, Instr);
2882
2883 // Does this instruction return a value ?
2884 bool IsVoidRetTy = Instr->getType()->isVoidTy();
2885
2886 Instruction *Cloned = Instr->clone();
2887 if (!IsVoidRetTy)
2888 Cloned->setName(Instr->getName() + ".cloned");
2889
2890 // Replace the operands of the cloned instructions with their scalar
2891 // equivalents in the new loop.
2892 for (unsigned op = 0, e = User.getNumOperands(); op != e; ++op) {
2893 auto *Operand = dyn_cast<Instruction>(Instr->getOperand(op));
2894 auto InputInstance = Instance;
2895 if (!Operand || !OrigLoop->contains(Operand) ||
2896 (Cost->isUniformAfterVectorization(Operand, State.VF)))
2897 InputInstance.Lane = 0;
2898 auto *NewOp = State.get(User.getOperand(op), InputInstance);
2899 Cloned->setOperand(op, NewOp);
2900 }
2901 addNewMetadata(Cloned, Instr);
2902
2903 // Place the cloned scalar in the new loop.
2904 Builder.Insert(Cloned);
2905
2906 // TODO: Set result for VPValue of VPReciplicateRecipe. This requires
2907 // representing scalar values in VPTransformState. Add the cloned scalar to
2908 // the scalar map entry.
2909 VectorLoopValueMap.setScalarValue(Instr, Instance, Cloned);
2910
2911 // If we just cloned a new assumption, add it the assumption cache.
2912 if (auto *II = dyn_cast<IntrinsicInst>(Cloned))
2913 if (II->getIntrinsicID() == Intrinsic::assume)
2914 AC->registerAssumption(II);
2915
2916 // End if-block.
2917 if (IfPredicateInstr)
2918 PredicatedInstructions.push_back(Cloned);
2919}
2920
2921PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start,
2922 Value *End, Value *Step,
2923 Instruction *DL) {
2924 BasicBlock *Header = L->getHeader();
2925 BasicBlock *Latch = L->getLoopLatch();
2926 // As we're just creating this loop, it's possible no latch exists
2927 // yet. If so, use the header as this will be a single block loop.
2928 if (!Latch)
2929 Latch = Header;
2930
2931 IRBuilder<> Builder(&*Header->getFirstInsertionPt());
2932 Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction);
2933 setDebugLocFromInst(Builder, OldInst);
2934 auto *Induction = Builder.CreatePHI(Start->getType(), 2, "index");
2935
2936 Builder.SetInsertPoint(Latch->getTerminator());
2937 setDebugLocFromInst(Builder, OldInst);
2938
2939 // Create i+1 and fill the PHINode.
2940 Value *Next = Builder.CreateAdd(Induction, Step, "index.next");
2941 Induction->addIncoming(Start, L->getLoopPreheader());
2942 Induction->addIncoming(Next, Latch);
2943 // Create the compare.
2944 Value *ICmp = Builder.CreateICmpEQ(Next, End);
2945 Builder.CreateCondBr(ICmp, L->getUniqueExitBlock(), Header);
2946
2947 // Now we have two terminators. Remove the old one from the block.
2948 Latch->getTerminator()->eraseFromParent();
2949
2950 return Induction;
2951}
2952
2953Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
2954 if (TripCount)
2955 return TripCount;
2956
2957 assert(L && "Create Trip Count for null loop.");
2958 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
2959 // Find the loop boundaries.
2960 ScalarEvolution *SE = PSE.getSE();
2961 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
2962 assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
2963 "Invalid loop count");
2964
2965 Type *IdxTy = Legal->getWidestInductionType();
2966 assert(IdxTy && "No type for induction");
2967
2968 // The exit count might have the type of i64 while the phi is i32. This can
2969 // happen if we have an induction variable that is sign extended before the
2970 // compare. The only way that we get a backedge taken count is that the
2971 // induction variable was signed and as such will not overflow. In such a case
2972 // truncation is legal.
2973 if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) >
2974 IdxTy->getPrimitiveSizeInBits())
2975 BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
2976 BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
2977
2978 // Get the total trip count from the count by adding 1.
2979 const SCEV *ExitCount = SE->getAddExpr(
2980 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
2981
2982 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
2983
2984 // Expand the trip count and place the new instructions in the preheader.
2985 // Notice that the pre-header does not change, only the loop body.
2986 SCEVExpander Exp(*SE, DL, "induction");
2987
2988 // Count holds the overall loop count (N).
2989 TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
2990 L->getLoopPreheader()->getTerminator());
2991
2992 if (TripCount->getType()->isPointerTy())
2993 TripCount =
2994 CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",
2995 L->getLoopPreheader()->getTerminator());
2996
2997 return TripCount;
2998}
2999
3000Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
3001 if (VectorTripCount)
3002 return VectorTripCount;
3003
3004 Value *TC = getOrCreateTripCount(L);
3005 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
3006
3007 Type *Ty = TC->getType();
3008 // This is where we can make the step a runtime constant.
3009 Value *Step = createStepForVF(Builder, ConstantInt::get(Ty, UF), VF);
3010
3011 // If the tail is to be folded by masking, round the number of iterations N
3012 // up to a multiple of Step instead of rounding down. This is done by first
3013 // adding Step-1 and then rounding down. Note that it's ok if this addition
3014 // overflows: the vector induction variable will eventually wrap to zero given
3015 // that it starts at zero and its Step is a power of two; the loop will then
3016 // exit, with the last early-exit vector comparison also producing all-true.
3017 if (Cost->foldTailByMasking()) {
3018 assert(isPowerOf2_32(VF.getKnownMinValue() * UF) &&
3019 "VF*UF must be a power of 2 when folding tail by masking");
3020 assert(!VF.isScalable() &&
3021 "Tail folding not yet supported for scalable vectors");
3022 TC = Builder.CreateAdd(
3023 TC, ConstantInt::get(Ty, VF.getKnownMinValue() * UF - 1), "n.rnd.up");
3024 }
3025
3026 // Now we need to generate the expression for the part of the loop that the
3027 // vectorized body will execute. This is equal to N - (N % Step) if scalar
3028 // iterations are not required for correctness, or N - Step, otherwise. Step
3029 // is equal to the vectorization factor (number of SIMD elements) times the
3030 // unroll factor (number of SIMD instructions).
3031 Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");
3032
3033 // There are two cases where we need to ensure (at least) the last iteration
3034 // runs in the scalar remainder loop. Thus, if the step evenly divides
3035 // the trip count, we set the remainder to be equal to the step. If the step
3036 // does not evenly divide the trip count, no adjustment is necessary since
3037 // there will already be scalar iterations. Note that the minimum iterations
3038 // check ensures that N >= Step. The cases are:
3039 // 1) If there is a non-reversed interleaved group that may speculatively
3040 // access memory out-of-bounds.
3041 // 2) If any instruction may follow a conditionally taken exit. That is, if
3042 // the loop contains multiple exiting blocks, or a single exiting block
3043 // which is not the latch.
3044 if (VF.isVector() && Cost->requiresScalarEpilogue()) {
3045 auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));
3046 R = Builder.CreateSelect(IsZero, Step, R);
3047 }
3048
3049 VectorTripCount = Builder.CreateSub(TC, R, "n.vec");
3050
3051 return VectorTripCount;
3052}
3053
3054Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
3055 const DataLayout &DL) {
3056 // Verify that V is a vector type with same number of elements as DstVTy.
3057 auto *DstFVTy = cast<FixedVectorType>(DstVTy);
3058 unsigned VF = DstFVTy->getNumElements();
3059 auto *SrcVecTy = cast<FixedVectorType>(V->getType());
3060 assert((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match");
3061 Type *SrcElemTy = SrcVecTy->getElementType();
3062 Type *DstElemTy = DstFVTy->getElementType();
3063 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&
3064 "Vector elements must have same size");
3065
3066 // Do a direct cast if element types are castable.
3067 if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
3068 return Builder.CreateBitOrPointerCast(V, DstFVTy);
3069 }
3070 // V cannot be directly casted to desired vector type.
3071 // May happen when V is a floating point vector but DstVTy is a vector of
3072 // pointers or vice-versa. Handle this using a two-step bitcast using an
3073 // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
3074 assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&
3075 "Only one type should be a pointer type");
3076 assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&
3077 "Only one type should be a floating point type");
3078 Type *IntTy =
3079 IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
3080 auto *VecIntTy = FixedVectorType::get(IntTy, VF);
3081 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
3082 return Builder.CreateBitOrPointerCast(CastVal, DstFVTy);
3083}
3084
3085void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
3086 BasicBlock *Bypass) {
3087 Value *Count = getOrCreateTripCount(L);
3088 // Reuse existing vector loop preheader for TC checks.
3089 // Note that new preheader block is generated for vector loop.
3090 BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
3091 IRBuilder<> Builder(TCCheckBlock->getTerminator());
3092
3093 // Generate code to check if the loop's trip count is less than VF * UF, or
3094 // equal to it in case a scalar epilogue is required; this implies that the
3095 // vector trip count is zero. This check also covers the case where adding one
3096 // to the backedge-taken count overflowed leading to an incorrect trip count
3097 // of zero. In this case we will also jump to the scalar loop.
3098 auto P = Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE
3099 : ICmpInst::ICMP_ULT;
3100
3101 // If tail is to be folded, vector loop takes care of all iterations.
3102 Value *CheckMinIters = Builder.getFalse();
3103 if (!Cost->foldTailByMasking()) {
3104 Value *Step =
3105 createStepForVF(Builder, ConstantInt::get(Count->getType(), UF), VF);
3106 CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
3107 }
3108 // Create new preheader for vector loop.
3109 LoopVectorPreHeader =
3110 SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(), DT, LI, nullptr,
3111 "vector.ph");
3112
3113 assert(DT->properlyDominates(DT->getNode(TCCheckBlock),
3114 DT->getNode(Bypass)->getIDom()) &&
3115 "TC check is expected to dominate Bypass");
3116
3117 // Update dominator for Bypass & LoopExit.
3118 DT->changeImmediateDominator(Bypass, TCCheckBlock);
3119 DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);
3120
3121 ReplaceInstWithInst(
3122 TCCheckBlock->getTerminator(),
3123 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
3124 LoopBypassBlocks.push_back(TCCheckBlock);
3125}
3126
3127void InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {
3128 // Reuse existing vector loop preheader for SCEV checks.
3129 // Note that new preheader block is generated for vector loop.
3130 BasicBlock *const SCEVCheckBlock = LoopVectorPreHeader;
3131
3132 // Generate the code to check that the SCEV assumptions that we made.
3133 // We want the new basic block to start at the first instruction in a
3134 // sequence of instructions that form a check.
3135 SCEVExpander Exp(*PSE.getSE(), Bypass->getModule()->getDataLayout(),
3136 "scev.check");
3137 Value *SCEVCheck = Exp.expandCodeForPredicate(
3138 &PSE.getUnionPredicate(), SCEVCheckBlock->getTerminator());
3139
3140 if (auto *C = dyn_cast<ConstantInt>(SCEVCheck))
3141 if (C->isZero())
3142 return;
3143
3144 assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||
3145 (OptForSizeBasedOnProfile &&
3146 Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
3147 "Cannot SCEV check stride or overflow when optimizing for size");
3148
3149 SCEVCheckBlock->setName("vector.scevcheck");
3150 // Create new preheader for vector loop.
3151 LoopVectorPreHeader =
3152 SplitBlock(SCEVCheckBlock, SCEVCheckBlock->getTerminator(), DT, LI,
3153 nullptr, "vector.ph");
3154
3155 // Update dominator only if this is first RT check.
3156 if (LoopBypassBlocks.empty()) {
3157 DT->changeImmediateDominator(Bypass, SCEVCheckBlock);
3158 DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock);
3159 }
3160
3161 ReplaceInstWithInst(
3162 SCEVCheckBlock->getTerminator(),
3163 BranchInst::Create(Bypass, LoopVectorPreHeader, SCEVCheck));
3164 LoopBypassBlocks.push_back(SCEVCheckBlock);
3165 AddedSafetyChecks = true;
3166}
3167
3168void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass) {
3169 // VPlan-native path does not do any analysis for runtime checks currently.
3170 if (EnableVPlanNativePath)
3171 return;
3172
3173 // Reuse existing vector loop preheader for runtime memory checks.
3174 // Note that new preheader block is generated for vector loop.
3175 BasicBlock *const MemCheckBlock = L->getLoopPreheader();
3176
3177 // Generate the code that checks in runtime if arrays overlap. We put the
3178 // checks into a separate block to make the more common case of few elements
3179 // faster.
3180 auto *LAI = Legal->getLAI();
3181 const auto &RtPtrChecking = *LAI->getRuntimePointerChecking();
3182 if (!RtPtrChecking.Need)
3183 return;
3184
3185 if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
3186 assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
3187 "Cannot emit memory checks when optimizing for size, unless forced "
3188 "to vectorize.");
3189 ORE->emit([&]() {
3190 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
3191 L->getStartLoc(), L->getHeader())
3192 << "Code-size may be reduced by not forcing "
3193 "vectorization, or by source-code modifications "
3194 "eliminating the need for runtime checks "
3195 "(e.g., adding 'restrict').";
3196 });
3197 }
3198
3199 MemCheckBlock->setName("vector.memcheck");
3200 // Create new preheader for vector loop.
3201 LoopVectorPreHeader =
3202 SplitBlock(MemCheckBlock, MemCheckBlock->getTerminator(), DT, LI, nullptr,
3203 "vector.ph");
3204
3205 auto *CondBranch = cast<BranchInst>(
3206 Builder.CreateCondBr(Builder.getTrue(), Bypass, LoopVectorPreHeader));
3207 ReplaceInstWithInst(MemCheckBlock->getTerminator(), CondBranch);
3208 LoopBypassBlocks.push_back(MemCheckBlock);
3209 AddedSafetyChecks = true;
3210
3211 // Update dominator only if this is first RT check.
3212 if (LoopBypassBlocks.empty()) {
3213 DT->changeImmediateDominator(Bypass, MemCheckBlock);
3214 DT->changeImmediateDominator(LoopExitBlock, MemCheckBlock);
3215 }
3216
3217 Instruction *FirstCheckInst;
3218 Instruction *MemRuntimeCheck;
3219 std::tie(FirstCheckInst, MemRuntimeCheck) =
3220 addRuntimeChecks(MemCheckBlock->getTerminator(), OrigLoop,
3221 RtPtrChecking.getChecks(), RtPtrChecking.getSE());
3222 assert(MemRuntimeCheck && "no RT checks generated although RtPtrChecking "
3223 "claimed checks are required");
3224 CondBranch->setCondition(MemRuntimeCheck);
3225
3226 // We currently don't use LoopVersioning for the actual loop cloning but we
3227 // still use it to add the noalias metadata.
3228 LVer = std::make_unique<LoopVersioning>(
3229 *Legal->getLAI(),
3230 Legal->getLAI()->getRuntimePointerChecking()->getChecks(), OrigLoop, LI,
3231 DT, PSE.getSE());
3232 LVer->prepareNoAliasMetadata();
3233}
3234
3235Value *InnerLoopVectorizer::emitTransformedIndex(
3236 IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL,
3237 const InductionDescriptor &ID) const {
3238
3239 SCEVExpander Exp(*SE, DL, "induction");
3240 auto Step = ID.getStep();
3241 auto StartValue = ID.getStartValue();
3242 assert(Index->getType() == Step->getType() &&
3243 "Index type does not match StepValue type");
3244
3245 // Note: the IR at this point is broken. We cannot use SE to create any new
3246 // SCEV and then expand it, hoping that SCEV's simplification will give us
3247 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
3248 // lead to various SCEV crashes. So all we can do is to use builder and rely
3249 // on InstCombine for future simplifications. Here we handle some trivial
3250 // cases only.
3251 auto CreateAdd = [&B](Value *X, Value *Y) {
3252 assert(X->getType() == Y->getType() && "Types don't match!");
3253 if (auto *CX = dyn_cast<ConstantInt>(X))
3254 if (CX->isZero())
3255 return Y;
3256 if (auto *CY = dyn_cast<ConstantInt>(Y))
3257 if (CY->isZero())
3258 return X;
3259 return B.CreateAdd(X, Y);
3260 };
3261
3262 auto CreateMul = [&B](Value *X, Value *Y) {
3263 assert(X->getType() == Y->getType() && "Types don't match!");
3264 if (auto *CX = dyn_cast<ConstantInt>(X))
3265 if (CX->isOne())
3266 return Y;
3267 if (auto *CY = dyn_cast<ConstantInt>(Y))
3268 if (CY->isOne())
3269 return X;
3270 return B.CreateMul(X, Y);
3271 };
3272
3273 // Get a suitable insert point for SCEV expansion. For blocks in the vector
3274 // loop, choose the end of the vector loop header (=LoopVectorBody), because
3275 // the DomTree is not kept up-to-date for additional blocks generated in the
3276 // vector loop. By using the header as insertion point, we guarantee that the
3277 // expanded instructions dominate all their uses.
3278 auto GetInsertPoint = [this, &B]() {
3279 BasicBlock *InsertBB = B.GetInsertPoint()->getParent();
3280 if (InsertBB != LoopVectorBody &&
3281 LI->getLoopFor(LoopVectorBody) == LI->getLoopFor(InsertBB))
3282 return LoopVectorBody->getTerminator();
3283 return &*B.GetInsertPoint();
3284 };
3285 switch (ID.getKind()) {
3286 case InductionDescriptor::IK_IntInduction: {
3287 assert(Index->getType() == StartValue->getType() &&
3288 "Index type does not match StartValue type");
3289 if (ID.getConstIntStepValue() && ID.getConstIntStepValue()->isMinusOne())
3290 return B.CreateSub(StartValue, Index);
3291 auto *Offset = CreateMul(
3292 Index, Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint()));
3293 return CreateAdd(StartValue, Offset);
3294 }
3295 case InductionDescriptor::IK_PtrInduction: {
3296 assert(isa<SCEVConstant>(Step) &&
3297 "Expected constant step for pointer induction");
3298 return B.CreateGEP(
3299 StartValue->getType()->getPointerElementType(), StartValue,
3300 CreateMul(Index,
3301 Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint())));
3302 }
3303 case InductionDescriptor::IK_FpInduction: {
3304 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value");
3305 auto InductionBinOp = ID.getInductionBinOp();
3306 assert(InductionBinOp &&
3307 (InductionBinOp->getOpcode() == Instruction::FAdd ||
3308 InductionBinOp->getOpcode() == Instruction::FSub) &&
3309 "Original bin op should be defined for FP induction");
3310
3311 Value *StepValue = cast<SCEVUnknown>(Step)->getValue();
3312
3313 // Floating point operations had to be 'fast' to enable the induction.
3314 FastMathFlags Flags;
3315 Flags.setFast();
3316
3317 Value *MulExp = B.CreateFMul(StepValue, Index);
3318 if (isa<Instruction>(MulExp))
3319 // We have to check, the MulExp may be a constant.
3320 cast<Instruction>(MulExp)->setFastMathFlags(Flags);
3321
3322 Value *BOp = B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
3323 "induction");
3324 if (isa<Instruction>(BOp))
3325 cast<Instruction>(BOp)->setFastMathFlags(Flags);
3326
3327 return BOp;
3328 }
3329 case InductionDescriptor::IK_NoInduction:
3330 return nullptr;
3331 }
3332 llvm_unreachable("invalid enum");
3333}
3334
3335Loop *InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
3336 LoopScalarBody = OrigLoop->getHeader();
3337 LoopVectorPreHeader = OrigLoop->getLoopPreheader();
3338 LoopExitBlock = OrigLoop->getUniqueExitBlock();
3339 assert(LoopExitBlock && "Must have an exit block");
3340 assert(LoopVectorPreHeader && "Invalid loop structure");
3341
3342 LoopMiddleBlock =
3343 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3344 LI, nullptr, Twine(Prefix) + "middle.block");
3345 LoopScalarPreHeader =
3346 SplitBlock(LoopMiddleBlock, LoopMiddleBlock->getTerminator(), DT, LI,
3347 nullptr, Twine(Prefix) + "scalar.ph");
3348
3349 // Set up branch from middle block to the exit and scalar preheader blocks.
3350 // completeLoopSkeleton will update the condition to use an iteration check,
3351 // if required to decide whether to execute the remainder.
3352 BranchInst *BrInst =
3353 BranchInst::Create(LoopExitBlock, LoopScalarPreHeader, Builder.getTrue());
3354 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
3355 BrInst->setDebugLoc(ScalarLatchTerm->getDebugLoc());
3356 ReplaceInstWithInst(LoopMiddleBlock->getTerminator(), BrInst);
3357
3358 // We intentionally don't let SplitBlock to update LoopInfo since
3359 // LoopVectorBody should belong to another loop than LoopVectorPreHeader.
3360 // LoopVectorBody is explicitly added to the correct place few lines later.
3361 LoopVectorBody =
3362 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3363 nullptr, nullptr, Twine(Prefix) + "vector.body");
3364
3365 // Update dominator for loop exit.
3366 DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock);
3367
3368 // Create and register the new vector loop.
3369 Loop *Lp = LI->AllocateLoop();
3370 Loop *ParentLoop = OrigLoop->getParentLoop();
3371
3372 // Insert the new loop into the loop nest and register the new basic blocks
3373 // before calling any utilities such as SCEV that require valid LoopInfo.
3374 if (ParentLoop) {
3375 ParentLoop->addChildLoop(Lp);
3376 } else {
3377 LI->addTopLevelLoop(Lp);
3378 }
3379 Lp->addBasicBlockToLoop(LoopVectorBody, *LI);
3380 return Lp;
3381}
3382
3383void InnerLoopVectorizer::createInductionResumeValues(
3384 Loop *L, Value *VectorTripCount,
3385 std::pair<BasicBlock *, Value *> AdditionalBypass) {
3386 assert(VectorTripCount && L && "Expected valid arguments");
3387 assert(((AdditionalBypass.first && AdditionalBypass.second) ||
3388 (!AdditionalBypass.first && !AdditionalBypass.second)) &&
3389 "Inconsistent information about additional bypass.");
3390 // We are going to resume the execution of the scalar loop.
3391 // Go over all of the induction variables that we found and fix the
3392 // PHIs that are left in the scalar version of the loop.
3393 // The starting values of PHI nodes depend on the counter of the last
3394 // iteration in the vectorized loop.
3395 // If we come from a bypass edge then we need to start from the original
3396 // start value.
3397 for (auto &InductionEntry : Legal->getInductionVars()) {
3398 PHINode *OrigPhi = InductionEntry.first;
3399 InductionDescriptor II = InductionEntry.second;
3400
3401 // Create phi nodes to merge from the backedge-taken check block.
3402 PHINode *BCResumeVal =
3403 PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val",
3404 LoopScalarPreHeader->getTerminator());
3405 // Copy original phi DL over to the new one.
3406 BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
3407 Value *&EndValue = IVEndValues[OrigPhi];
3408 Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
3409 if (OrigPhi == OldInduction) {
3410 // We know what the end value is.
3411 EndValue = VectorTripCount;
3412 } else {
3413 IRBuilder<> B(L->getLoopPreheader()->getTerminator());
3414 Type *StepType = II.getStep()->getType();
3415 Instruction::CastOps CastOp =
3416 CastInst::getCastOpcode(VectorTripCount, true, StepType, true);
3417 Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd");
3418 const DataLayout &DL = LoopScalarBody->getModule()->getDataLayout();
3419 EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
3420 EndValue->setName("ind.end");
3421
3422 // Compute the end value for the additional bypass (if applicable).
3423 if (AdditionalBypass.first) {
3424 B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt()));
3425 CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true,
3426 StepType, true);
3427 CRD =
3428 B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd");
3429 EndValueFromAdditionalBypass =
3430 emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
3431 EndValueFromAdditionalBypass->setName("ind.end");
3432 }
3433 }
3434 // The new PHI merges the original incoming value, in case of a bypass,
3435 // or the value at the end of the vectorized loop.
3436 BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);
3437
3438 // Fix the scalar body counter (PHI node).
3439 // The old induction's phi node in the scalar body needs the truncated
3440 // value.
3441 for (BasicBlock *BB : LoopBypassBlocks)
3442 BCResumeVal->addIncoming(II.getStartValue(), BB);
3443
3444 if (AdditionalBypass.first)
3445 BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first,
3446 EndValueFromAdditionalBypass);
3447
3448 OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
3449 }
3450}
3451
3452BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(Loop *L,
3453 MDNode *OrigLoopID) {
3454 assert(L && "Expected valid loop.");
3455
3456 // The trip counts should be cached by now.
3457 Value *Count = getOrCreateTripCount(L);
3458 Value *VectorTripCount = getOrCreateVectorTripCount(L);
3459
3460 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
3461
3462 // Add a check in the middle block to see if we have completed
3463 // all of the iterations in the first vector loop.
3464 // If (N - N%VF) == N, then we *don't* need to run the remainder.
3465 // If tail is to be folded, we know we don't need to run the remainder.
3466 if (!Cost->foldTailByMasking()) {
3467 Instruction *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ,
3468 Count, VectorTripCount, "cmp.n",
3469 LoopMiddleBlock->getTerminator());
3470
3471 // Here we use the same DebugLoc as the scalar loop latch terminator instead
3472 // of the corresponding compare because they may have ended up with
3473 // different line numbers and we want to avoid awkward line stepping while
3474 // debugging. Eg. if the compare has got a line number inside the loop.
3475 CmpN->setDebugLoc(ScalarLatchTerm->getDebugLoc());
3476 cast<BranchInst>(LoopMiddleBlock->getTerminator())->setCondition(CmpN);
3477 }
3478
3479 // Get ready to start creating new instructions into the vectorized body.
3480 assert(LoopVectorPreHeader == L->getLoopPreheader() &&
3481 "Inconsistent vector loop preheader");
3482 Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt());
3483
3484 Optional<MDNode *> VectorizedLoopID =
3485 makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
3486 LLVMLoopVectorizeFollowupVectorized});
3487 if (VectorizedLoopID.hasValue()) {
3488 L->setLoopID(VectorizedLoopID.getValue());
3489
3490 // Do not setAlreadyVectorized if loop attributes have been defined
3491 // explicitly.
3492 return LoopVectorPreHeader;
3493 }
3494
3495 // Keep all loop hints from the original loop on the vector loop (we'll
3496 // replace the vectorizer-specific hints below).
3497 if (MDNode *LID = OrigLoop->getLoopID())
3498 L->setLoopID(LID);
3499
3500 LoopVectorizeHints Hints(L, true, *ORE);
3501 Hints.setAlreadyVectorized();
3502
3503#ifdef EXPENSIVE_CHECKS
3504 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
3505 LI->verify(*DT);
3506#endif
3507
3508 return LoopVectorPreHeader;
3509}
3510
3511BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
3512 /*
3513 In this function we generate a new loop. The new loop will contain
3514 the vectorized instructions while the old loop will continue to run the
3515 scalar remainder.
3516
3517 [ ] <-- loop iteration number check.
3518 / |
3519 / v
3520 | [ ] <-- vector loop bypass (may consist of multiple blocks).
3521 | / |
3522 | / v
3523 || [ ] <-- vector pre header.
3524 |/ |
3525 | v
3526 | [ ] \
3527 | [ ]_| <-- vector loop.
3528 | |
3529 | v
3530 | -[ ] <--- middle-block.
3531 | / |
3532 | / v
3533 -|- >[ ] <--- new preheader.
3534 | |
3535 | v
3536 | [ ] \
3537 | [ ]_| <-- old scalar loop to handle remainder.
3538 \ |
3539 \ v
3540 >[ ] <-- exit block.
3541 ...
3542 */
3543
3544 // Get the metadata of the original loop before it gets modified.
3545 MDNode *OrigLoopID = OrigLoop->getLoopID();
3546
3547 // Create an empty vector loop, and prepare basic blocks for the runtime
3548 // checks.
3549 Loop *Lp = createVectorLoopSkeleton("");
3550
3551 // Now, compare the new count to zero. If it is zero skip the vector loop and
3552 // jump to the scalar loop. This check also covers the case where the
3553 // backedge-taken count is uint##_max: adding one to it will overflow leading
3554 // to an incorrect trip count of zero. In this (rare) case we will also jump
3555 // to the scalar loop.
3556 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader);
3557
3558 // Generate the code to check any assumptions that we've made for SCEV
3559 // expressions.
3560 emitSCEVChecks(Lp, LoopScalarPreHeader);
3561
3562 // Generate the code that checks in runtime if arrays overlap. We put the
3563 // checks into a separate block to make the more common case of few elements
3564 // faster.
3565 emitMemRuntimeChecks(Lp, LoopScalarPreHeader);
3566
3567 // Some loops have a single integer induction variable, while other loops
3568 // don't. One example is c++ iterators that often have multiple pointer
3569 // induction variables. In the code below we also support a case where we
3570 // don't have a single induction variable.
3571 //
3572 // We try to obtain an induction variable from the original loop as hard
3573 // as possible. However if we don't find one that:
3574 // - is an integer
3575 // - counts from zero, stepping by one
3576 // - is the size of the widest induction variable type
3577 // then we create a new one.
3578 OldInduction = Legal->getPrimaryInduction();
3579 Type *IdxTy = Legal->getWidestInductionType();
3580 Value *StartIdx = ConstantInt::get(IdxTy, 0);
3581 // The loop step is equal to the vectorization factor (num of SIMD elements)
3582 // times the unroll factor (num of SIMD instructions).
3583 Builder.SetInsertPoint(&*Lp->getHeader()->getFirstInsertionPt());
3584 Value *Step = createStepForVF(Builder, ConstantInt::get(IdxTy, UF), VF);
3585 Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
3586 Induction =
3587 createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
3588 getDebugLocFromInstOrOperands(OldInduction));
3589
3590 // Emit phis for the new starting index of the scalar loop.
3591 createInductionResumeValues(Lp, CountRoundDown);
3592
3593 return completeLoopSkeleton(Lp, OrigLoopID);
3594}
3595
3596// Fix up external users of the induction variable. At this point, we are
3597// in LCSSA form, with all external PHIs that use the IV having one input value,
3598// coming from the remainder loop. We need those PHIs to also have a correct
3599// value for the IV when arriving directly from the middle block.
3600void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
3601 const InductionDescriptor &II,
3602 Value *CountRoundDown, Value *EndValue,
3603 BasicBlock *MiddleBlock) {
3604 // There are two kinds of external IV usages - those that use the value
3605 // computed in the last iteration (the PHI) and those that use the penultimate
3606 // value (the value that feeds into the phi from the loop latch).
3607 // We allow both, but they, obviously, have different values.
3608
3609 assert(OrigLoop->getUniqueExitBlock() && "Expected a single exit block");
3610
3611 DenseMap<Value *, Value *> MissingVals;
3612
3613 // An external user of the last iteration's value should see the value that
3614 // the remainder loop uses to initialize its own IV.
3615 Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch());
3616 for (User *U : PostInc->users()) {
3617 Instruction *UI = cast<Instruction>(U);
3618 if (!OrigLoop->contains(UI)) {
3619 assert(isa<PHINode>(UI) && "Expected LCSSA form");
3620 MissingVals[UI] = EndValue;
3621 }
3622 }
3623
3624 // An external user of the penultimate value need to see EndValue - Step.
3625 // The simplest way to get this is to recompute it from the constituent SCEVs,
3626 // that is Start + (Step * (CRD - 1)).
3627 for (User *U : OrigPhi->users()) {
3628 auto *UI = cast<Instruction>(U);
3629 if (!OrigLoop->contains(UI)) {
3630 const DataLayout &DL =
3631 OrigLoop->getHeader()->getModule()->getDataLayout();
3632 assert(isa<PHINode>(UI) && "Expected LCSSA form");
3633
3634 IRBuilder<> B(MiddleBlock->getTerminator());
3635 Value *CountMinusOne = B.CreateSub(
3636 CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1));
3637 Value *CMO =
3638 !II.getStep()->getType()->isIntegerTy()
3639 ? B.CreateCast(Instruction::SIToFP, CountMinusOne,
3640 II.getStep()->getType())
3641 : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());
3642 CMO->setName("cast.cmo");
3643 Value *Escape = emitTransformedIndex(B, CMO, PSE.getSE(), DL, II);
3644 Escape->setName("ind.escape");
3645 MissingVals[UI] = Escape;
3646 }
3647 }
3648
3649 for (auto &I : MissingVals) {
3650 PHINode *PHI = cast<PHINode>(I.first);
3651 // One corner case we have to handle is two IVs "chasing" each-other,
3652 // that is %IV2 = phi [...], [ %IV1, %latch ]
3653 // In this case, if IV1 has an external use, we need to avoid adding both
3654 // "last value of IV1" and "penultimate value of IV2". So, verify that we
3655 // don't already have an incoming value for the middle block.
3656 if (PHI->getBasicBlockIndex(MiddleBlock) == -1)
3657 PHI->addIncoming(I.second, MiddleBlock);
3658 }
3659}
3660
3661namespace {
3662
3663struct CSEDenseMapInfo {
3664 static bool canHandle(const Instruction *I) {
3665 return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
3666 isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);
3667 }
3668
3669 static inline Instruction *getEmptyKey() {
3670 return DenseMapInfo<Instruction *>::getEmptyKey();
3671 }
3672
3673 static inline Instruction *getTombstoneKey() {
3674 return DenseMapInfo<Instruction *>::getTombstoneKey();
3675 }
3676
3677 static unsigned getHashValue(const Instruction *I) {
3678 assert(canHandle(I) && "Unknown instruction!");
3679 return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),
3680 I->value_op_end()));
3681 }
3682
3683 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
3684 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
3685 LHS == getTombstoneKey() || RHS == getTombstoneKey())
3686 return LHS == RHS;
3687 return LHS->isIdenticalTo(RHS);
3688 }
3689};
3690
3691} // end anonymous namespace
3692
3693///Perform cse of induction variable instructions.
3694static void cse(BasicBlock *BB) {
3695 // Perform simple cse.
3696 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
3697 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
3698 Instruction *In = &*I++;
3699
3700 if (!CSEDenseMapInfo::canHandle(In))
3701 continue;
3702
3703 // Check if we can replace this instruction with any of the
3704 // visited instructions.
3705 if (Instruction *V = CSEMap.lookup(In)) {
3706 In->replaceAllUsesWith(V);
3707 In->eraseFromParent();
3708 continue;
3709 }
3710
3711 CSEMap[In] = In;
3712 }
3713}
3714
3715InstructionCost
3716LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF,
3717 bool &NeedToScalarize) {
3718 assert(!VF.isScalable() && "scalable vectors not yet supported.");
3719 Function *F = CI->getCalledFunction();
3720 Type *ScalarRetTy = CI->getType();
3721 SmallVector<Type *, 4> Tys, ScalarTys;
3722 for (auto &ArgOp : CI->arg_operands())
3723 ScalarTys.push_back(ArgOp->getType());
3724
3725 // Estimate cost of scalarized vector call. The source operands are assumed
3726 // to be vectors, so we need to extract individual elements from there,
3727 // execute VF scalar calls, and then gather the result into the vector return
3728 // value.
3729 InstructionCost ScalarCallCost =
3730 TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys, TTI::TCK_RecipThroughput);
3731 if (VF.isScalar())
3732 return ScalarCallCost;
3733
3734 // Compute corresponding vector type for return value and arguments.
3735 Type *RetTy = ToVectorTy(ScalarRetTy, VF);
3736 for (Type *ScalarTy : ScalarTys)
3737 Tys.push_back(ToVectorTy(ScalarTy, VF));
3738
3739 // Compute costs of unpacking argument values for the scalar calls and
3740 // packing the return values to a vector.
3741 InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
3742
3743 InstructionCost Cost =
3744 ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
3745
3746 // If we can't emit a vector call for this function, then the currently found
3747 // cost is the cost we need to return.
3748 NeedToScalarize = true;
3749 VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
3750 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
3751
3752 if (!TLI || CI->isNoBuiltin() || !VecFunc)
3753 return Cost;
3754
3755 // If the corresponding vector cost is cheaper, return its cost.
3756 InstructionCost VectorCallCost =
3757 TTI.getCallInstrCost(nullptr, RetTy, Tys, TTI::TCK_RecipThroughput);
3758 if (VectorCallCost < Cost) {
3759 NeedToScalarize = false;
3760 Cost = VectorCallCost;
3761 }
3762 return Cost;
3763}
3764
3765InstructionCost
3766LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
3767 ElementCount VF) {
3768 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3769 assert(ID && "Expected intrinsic call!");
3770
3771 IntrinsicCostAttributes CostAttrs(ID, *CI, VF);
3772 return TTI.getIntrinsicInstrCost(CostAttrs,
3773 TargetTransformInfo::TCK_RecipThroughput);
3774}
3775
3776static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
3777 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3778 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3779 return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
3780}
3781
3782static Type *largestIntegerVectorType(Type *T1, Type *T2) {
3783 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3784 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3785 return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
3786}
3787
3788void InnerLoopVectorizer::truncateToMinimalBitwidths() {
3789 // For every instruction `I` in MinBWs, truncate the operands, create a
3790 // truncated version of `I` and reextend its result. InstCombine runs
3791 // later and will remove any ext/trunc pairs.
3792 SmallPtrSet<Value *, 4> Erased;
3793 for (const auto &KV : Cost->getMinimalBitwidths()) {
3794 // If the value wasn't vectorized, we must maintain the original scalar
3795 // type. The absence of the value from VectorLoopValueMap indicates that it
3796 // wasn't vectorized.
3797 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))
3798 continue;
3799 for (unsigned Part = 0; Part < UF; ++Part) {
3800 Value *I = getOrCreateVectorValue(KV.first, Part);
3801 if (Erased.count(I) || I->use_empty() || !isa<Instruction>(I))
3802 continue;
3803 Type *OriginalTy = I->getType();
3804 Type *ScalarTruncatedTy =
3805 IntegerType::get(OriginalTy->getContext(), KV.second);
3806 auto *TruncatedTy = FixedVectorType::get(
3807 ScalarTruncatedTy,
3808 cast<FixedVectorType>(OriginalTy)->getNumElements());
3809 if (TruncatedTy == OriginalTy)
3810 continue;
3811
3812 IRBuilder<> B(cast<Instruction>(I));
3813 auto ShrinkOperand = [&](Value *V) -> Value * {
3814 if (auto *ZI = dyn_cast<ZExtInst>(V))
3815 if (ZI->getSrcTy() == TruncatedTy)
3816 return ZI->getOperand(0);
3817 return B.CreateZExtOrTrunc(V, TruncatedTy);
3818 };
3819
3820 // The actual instruction modification depends on the instruction type,
3821 // unfortunately.
3822 Value *NewI = nullptr;
3823 if (auto *BO = dyn_cast<BinaryOperator>(I)) {
3824 NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)),
3825 ShrinkOperand(BO->getOperand(1)));
3826
3827 // Any wrapping introduced by shrinking this operation shouldn't be
3828 // considered undefined behavior. So, we can't unconditionally copy
3829 // arithmetic wrapping flags to NewI.
3830 cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false);
3831 } else if (auto *CI = dyn_cast<ICmpInst>(I)) {
3832 NewI =
3833 B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)),
3834 ShrinkOperand(CI->getOperand(1)));
3835 } else if (auto *SI = dyn_cast<SelectInst>(I)) {
3836 NewI = B.CreateSelect(SI->getCondition(),
3837 ShrinkOperand(SI->getTrueValue()),
3838 ShrinkOperand(SI->getFalseValue()));
3839 } else if (auto *CI = dyn_cast<CastInst>(I)) {
3840 switch (CI->getOpcode()) {
3841 default:
3842 llvm_unreachable("Unhandled cast!");
3843 case Instruction::Trunc:
3844 NewI = ShrinkOperand(CI->getOperand(0));
3845 break;
3846 case Instruction::SExt:
3847 NewI = B.CreateSExtOrTrunc(
3848 CI->getOperand(0),
3849 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3850 break;
3851 case Instruction::ZExt:
3852 NewI = B.CreateZExtOrTrunc(
3853 CI->getOperand(0),
3854 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3855 break;
3856 }
3857 } else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) {
3858 auto Elements0 = cast<FixedVectorType>(SI->getOperand(0)->getType())
3859 ->getNumElements();
3860 auto *O0 = B.CreateZExtOrTrunc(
3861 SI->getOperand(0),
3862 FixedVectorType::get(ScalarTruncatedTy, Elements0));
3863 auto Elements1 = cast<FixedVectorType>(SI->getOperand(1)->getType())
3864 ->getNumElements();
3865 auto *O1 = B.CreateZExtOrTrunc(
3866 SI->getOperand(1),
3867 FixedVectorType::get(ScalarTruncatedTy, Elements1));
3868
3869 NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
3870 } else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
3871 // Don't do anything with the operands, just extend the result.
3872 continue;
3873 } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
3874 auto Elements = cast<FixedVectorType>(IE->getOperand(0)->getType())
3875 ->getNumElements();
3876 auto *O0 = B.CreateZExtOrTrunc(
3877 IE->getOperand(0),
3878 FixedVectorType::get(ScalarTruncatedTy, Elements));
3879 auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy);
3880 NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2));
3881 } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
3882 auto Elements = cast<FixedVectorType>(EE->getOperand(0)->getType())
3883 ->getNumElements();
3884 auto *O0 = B.CreateZExtOrTrunc(
3885 EE->getOperand(0),
3886 FixedVectorType::get(ScalarTruncatedTy, Elements));
3887 NewI = B.CreateExtractElement(O0, EE->getOperand(2));
3888 } else {
3889 // If we don't know what to do, be conservative and don't do anything.
3890 continue;
3891 }
3892
3893 // Lastly, extend the result.
3894 NewI->takeName(cast<Instruction>(I));
3895 Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);
3896 I->replaceAllUsesWith(Res);
3897 cast<Instruction>(I)->eraseFromParent();
3898 Erased.insert(I);
3899 VectorLoopValueMap.resetVectorValue(KV.first, Part, Res);
3900 }
3901 }
3902
3903 // We'll have created a bunch of ZExts that are now parentless. Clean up.
3904 for (const auto &KV : Cost->getMinimalBitwidths()) {
3905 // If the value wasn't vectorized, we must maintain the original scalar
3906 // type. The absence of the value from VectorLoopValueMap indicates that it
3907 // wasn't vectorized.
3908 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))
3909 continue;
3910 for (unsigned Part = 0; Part < UF; ++Part) {
3911 Value *I = getOrCreateVectorValue(KV.first, Part);
3912 ZExtInst *Inst = dyn_cast<ZExtInst>(I);
3913 if (Inst && Inst->use_empty()) {
3914 Value *NewI = Inst->getOperand(0);
3915 Inst->eraseFromParent();
3916 VectorLoopValueMap.resetVectorValue(KV.first, Part, NewI);
3917 }
3918 }
3919 }
3920}
3921
3922void InnerLoopVectorizer::fixVectorizedLoop() {
3923 // Insert truncates and extends for any truncated instructions as hints to
3924 // InstCombine.
3925 if (VF.isVector())
3926 truncateToMinimalBitwidths();
3927
3928 // Fix widened non-induction PHIs by setting up the PHI operands.
3929 if (OrigPHIsToFix.size()) {
3930 assert(EnableVPlanNativePath &&
3931 "Unexpected non-induction PHIs for fixup in non VPlan-native path");
3932 fixNonInductionPHIs();
3933 }
3934
3935 // At this point every instruction in the original loop is widened to a
3936 // vector form. Now we need to fix the recurrences in the loop. These PHI
3937 // nodes are currently empty because we did not want to introduce cycles.
3938 // This is the second stage of vectorizing recurrences.
3939 fixCrossIterationPHIs();
3940
3941 // Forget the original basic block.
3942 PSE.getSE()->forgetLoop(OrigLoop);
3943
3944 // Fix-up external users of the induction variables.
3945 for (auto &Entry : Legal->getInductionVars())
3946 fixupIVUsers(Entry.first, Entry.second,
3947 getOrCreateVectorTripCount(LI->getLoopFor(LoopVectorBody)),
3948 IVEndValues[Entry.first], LoopMiddleBlock);
3949
3950 fixLCSSAPHIs();
3951 for (Instruction *PI : PredicatedInstructions)
3952 sinkScalarOperands(&*PI);
3953
3954 // Remove redundant induction instructions.
3955 cse(LoopVectorBody);
3956
3957 // Set/update profile weights for the vector and remainder loops as original
3958 // loop iterations are now distributed among them. Note that original loop
3959 // represented by LoopScalarBody becomes remainder loop after vectorization.
3960 //
3961 // For cases like foldTailByMasking() and requiresScalarEpiloque() we may
3962 // end up getting slightly roughened result but that should be OK since
3963 // profile is not inherently precise anyway. Note also possible bypass of
3964 // vector code caused by legality checks is ignored, assigning all the weight
3965 // to the vector loop, optimistically.
3966 //
3967 // For scalable vectorization we can't know at compile time how many iterations
3968 // of the loop are handled in one vector iteration, so instead assume a pessimistic
3969 // vscale of '1'.
3970 setProfileInfoAfterUnrolling(
3971 LI->getLoopFor(LoopScalarBody), LI->getLoopFor(LoopVectorBody),
3972 LI->getLoopFor(LoopScalarBody), VF.getKnownMinValue() * UF);
3973}
3974
3975void InnerLoopVectorizer::fixCrossIterationPHIs() {
3976 // In order to support recurrences we need to be able to vectorize Phi nodes.
3977 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
3978 // stage #2: We now need to fix the recurrences by adding incoming edges to
3979 // the currently empty PHI nodes. At this point every instruction in the
3980 // original loop is widened to a vector form so we can use them to construct
3981 // the incoming edges.
3982 for (PHINode &Phi : OrigLoop->getHeader()->phis()) {
3983 // Handle first-order recurrences and reductions that need to be fixed.
3984 if (Legal->isFirstOrderRecurrence(&Phi))
3985 fixFirstOrderRecurrence(&Phi);
3986 else if (Legal->isReductionVariable(&Phi))
3987 fixReduction(&Phi);
3988 }
3989}
3990
3991void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi) {
3992 // This is the second phase of vectorizing first-order recurrences. An
3993 // overview of the transformation is described below. Suppose we have the
3994 // following loop.
3995 //
3996 // for (int i = 0; i < n; ++i)
3997 // b[i] = a[i] - a[i - 1];
3998 //
3999 // There is a first-order recurrence on "a". For this loop, the shorthand
4000 // scalar IR looks like:
4001 //
4002 // scalar.ph:
4003 // s_init = a[-1]
4004 // br scalar.body
4005 //
4006 // scalar.body:
4007 // i = phi [0, scalar.ph], [i+1, scalar.body]
4008 // s1 = phi [s_init, scalar.ph], [s2, scalar.body]
4009 // s2 = a[i]
4010 // b[i] = s2 - s1
4011 // br cond, scalar.body, ...
4012 //
4013 // In this example, s1 is a recurrence because it's value depends on the
4014 // previous iteration. In the first phase of vectorization, we created a
4015 // temporary value for s1. We now complete the vectorization and produce the
4016 // shorthand vector IR shown below (for VF = 4, UF = 1).
4017 //
4018 // vector.ph:
4019 // v_init = vector(..., ..., ..., a[-1])
4020 // br vector.body
4021 //
4022 // vector.body
4023 // i = phi [0, vector.ph], [i+4, vector.body]
4024 // v1 = phi [v_init, vector.ph], [v2, vector.body]
4025 // v2 = a[i, i+1, i+2, i+3];
4026 // v3 = vector(v1(3), v2(0, 1, 2))
4027 // b[i, i+1, i+2, i+3] = v2 - v3
4028 // br cond, vector.body, middle.block
4029 //
4030 // middle.block:
4031 // x = v2(3)
4032 // br scalar.ph
4033 //
4034 // scalar.ph:
4035 // s_init = phi [x, middle.block], [a[-1], otherwise]
4036 // br scalar.body
4037 //
4038 // After execution completes the vector loop, we extract the next value of
4039 // the recurrence (x) to use as the initial value in the scalar loop.
4040
4041 // Get the original loop preheader and single loop latch.
4042 auto *Preheader = OrigLoop->getLoopPreheader();
4043 auto *Latch = OrigLoop->getLoopLatch();
4044
4045 // Get the initial and previous values of the scalar recurrence.
4046 auto *ScalarInit = Phi->getIncomingValueForBlock(Preheader);
4047 auto *Previous = Phi->getIncomingValueForBlock(Latch);
4048
4049 // Create a vector from the initial value.
4050 auto *VectorInit = ScalarInit;
4051 if (VF.isVector()) {
4052 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
4053 assert(!VF.isScalable() && "VF is assumed to be non scalable.");
4054 VectorInit = Builder.CreateInsertElement(
4055 PoisonValue::get(VectorType::get(VectorInit->getType(), VF)), VectorInit,
4056 Builder.getInt32(VF.getKnownMinValue() - 1), "vector.recur.init");
4057 }
4058
4059 // We constructed a temporary phi node in the first phase of vectorization.
4060 // This phi node will eventually be deleted.
4061 Builder.SetInsertPoint(
4062 cast<Instruction>(VectorLoopValueMap.getVectorValue(Phi, 0)));
4063
4064 // Create a phi node for the new recurrence. The current value will either be
4065 // the initial value inserted into a vector or loop-varying vector value.
4066 auto *VecPhi = Builder.CreatePHI(VectorInit->getType(), 2, "vector.recur");
4067 VecPhi->addIncoming(VectorInit, LoopVectorPreHeader);
4068
4069 // Get the vectorized previous value of the last part UF - 1. It appears last
4070 // among all unrolled iterations, due to the order of their construction.
4071 Value *PreviousLastPart = getOrCreateVectorValue(Previous, UF - 1);
4072
4073 // Find and set the insertion point after the previous value if it is an
4074 // instruction.
4075 BasicBlock::iterator InsertPt;
4076 // Note that the previous value may have been constant-folded so it is not
4077 // guaranteed to be an instruction in the vector loop.
4078 // FIXME: Loop invariant values do not form recurrences. We should deal with
4079 // them earlier.
4080 if (LI->getLoopFor(LoopVectorBody)->isLoopInvariant(PreviousLastPart))
4081 InsertPt = LoopVectorBody->getFirstInsertionPt();
4082 else {
4083 Instruction *PreviousInst = cast<Instruction>(PreviousLastPart);
4084 if (isa<PHINode>(PreviousLastPart))
4085 // If the previous value is a phi node, we should insert after all the phi
4086 // nodes in the block containing the PHI to avoid breaking basic block
4087 // verification. Note that the basic block may be different to
4088 // LoopVectorBody, in case we predicate the loop.
4089 InsertPt = PreviousInst->getParent()->getFirstInsertionPt();
4090 else
4091 InsertPt = ++PreviousInst->getIterator();
4092 }
4093 Builder.SetInsertPoint(&*InsertPt);
4094
4095 // We will construct a vector for the recurrence by combining the values for
4096 // the current and previous iterations. This is the required shuffle mask.
4097 assert(!VF.isScalable());
4098 SmallVector<int, 8> ShuffleMask(VF.getKnownMinValue());
4099 ShuffleMask[0] = VF.getKnownMinValue() - 1;
4100 for (unsigned I = 1; I < VF.getKnownMinValue(); ++I)
4101 ShuffleMask[I] = I + VF.getKnownMinValue() - 1;
4102
4103 // The vector from which to take the initial value for the current iteration
4104 // (actual or unrolled). Initially, this is the vector phi node.
4105 Value *Incoming = VecPhi;
4106
4107 // Shuffle the current and previous vector and update the vector parts.
4108 for (unsigned Part = 0; Part < UF; ++Part) {
4109 Value *PreviousPart = getOrCreateVectorValue(Previous, Part);
4110 Value *PhiPart = VectorLoopValueMap.getVectorValue(Phi, Part);
4111 auto *Shuffle =
4112 VF.isVector()
4113 ? Builder.CreateShuffleVector(Incoming, PreviousPart, ShuffleMask)
4114 : Incoming;
4115 PhiPart->replaceAllUsesWith(Shuffle);
4116 cast<Instruction>(PhiPart)->eraseFromParent();
4117 VectorLoopValueMap.resetVectorValue(Phi, Part, Shuffle);
4118 Incoming = PreviousPart;
4119 }
4120
4121 // Fix the latch value of the new recurrence in the vector loop.
4122 VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
4123
4124 // Extract the last vector element in the middle block. This will be the
4125 // initial value for the recurrence when jumping to the scalar loop.
4126 auto *ExtractForScalar = Incoming;
4127 if (VF.isVector()) {
4128 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
4129 ExtractForScalar = Builder.CreateExtractElement(
4130 ExtractForScalar, Builder.getInt32(VF.getKnownMinValue() - 1),
4131 "vector.recur.extract");
4132 }
4133 // Extract the second last element in the middle block if the
4134 // Phi is used outside the loop. We need to extract the phi itself
4135 // and not the last element (the phi update in the current iteration). This
4136 // will be the value when jumping to the exit block from the LoopMiddleBlock,
4137 // when the scalar loop is not run at all.
4138 Value *ExtractForPhiUsedOutsideLoop = nullptr;
4139 if (VF.isVector())
4140 ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement(
4141 Incoming, Builder.getInt32(VF.getKnownMinValue() - 2),
4142 "vector.recur.extract.for.phi");
4143 // When loop is unrolled without vectorizing, initialize
4144 // ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value of
4145 // `Incoming`. This is analogous to the vectorized case above: extracting the
4146 // second last element when VF > 1.
4147 else if (UF > 1)
4148 ExtractForPhiUsedOutsideLoop = getOrCreateVectorValue(Previous, UF - 2);
4149
4150 // Fix the initial value of the original recurrence in the scalar loop.
4151 Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());
4152 auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");
4153 for (auto *BB : predecessors(LoopScalarPreHeader)) {
4154 auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;
4155 Start->addIncoming(Incoming, BB);
4156 }
4157
4158 Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start);
4159 Phi->setName("scalar.recur");
4160
4161 // Finally, fix users of the recurrence outside the loop. The users will need
4162 // either the last value of the scalar recurrence or the last value of the
4163 // vector recurrence we extracted in the middle block. Since the loop is in
4164 // LCSSA form, we just need to find all the phi nodes for the original scalar
4165 // recurrence in the exit block, and then add an edge for the middle block.
4166 // Note that LCSSA does not imply single entry when the original scalar loop
4167 // had multiple exiting edges (as we always run the last iteration in the
4168 // scalar epilogue); in that case, the exiting path through middle will be
4169 // dynamically dead and the value picked for the phi doesn't matter.
4170 for (PHINode &LCSSAPhi : LoopExitBlock->phis())
4171 if (any_of(LCSSAPhi.incoming_values(),
4172 [Phi](Value *V) { return V == Phi; }))
4173 LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
4174}
4175
4176void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
4177 // Get it's reduction variable descriptor.
4178 assert(Legal->isReductionVariable(Phi) &&
4179 "Unable to find the reduction variable");
4180 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];
4181
4182 RecurKind RK = RdxDesc.getRecurrenceKind();
4183 TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
4184 Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
4185 setDebugLocFromInst(Builder, ReductionStartValue);
4186 bool IsInLoopReductionPhi = Cost->isInLoopReduction(Phi);
4187
4188 // This is the vector-clone of the value that leaves the loop.
4189 Type *VecTy = getOrCreateVectorValue(LoopExitInst, 0)->getType();
4190
4191 // Wrap flags are in general invalid after vectorization, clear them.
4192 clearReductionWrapFlags(RdxDesc);
4193
4194 // Fix the vector-loop phi.
4195
4196 // Reductions do not have to start at zero. They can start with
4197 // any loop invariant values.
4198 BasicBlock *Latch = OrigLoop->getLoopLatch();
4199 Value *LoopVal = Phi->getIncomingValueForBlock(Latch);
4200
4201 for (unsigned Part = 0; Part < UF; ++Part) {
4202 Value *VecRdxPhi = getOrCreateVectorValue(Phi, Part);
4203 Value *Val = getOrCreateVectorValue(LoopVal, Part);
4204 cast<PHINode>(VecRdxPhi)
4205 ->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
4206 }
4207
4208 // Before each round, move the insertion point right between
4209 // the PHIs and the values we are going to write.
4210 // This allows us to write both PHINodes and the extractelement
4211 // instructions.
4212 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
4213
4214 setDebugLocFromInst(Builder, LoopExitInst);
4215
4216 // If tail is folded by masking, the vector value to leave the loop should be
4217 // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
4218 // instead of the former. For an inloop reduction the reduction will already
4219 // be predicated, and does not need to be handled here.
4220 if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) {
4221 for (unsigned Part = 0; Part < UF; ++Part) {
4222 Value *VecLoopExitInst =
4223 VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
4224 Value *Sel = nullptr;
4225 for (User *U : VecLoopExitInst->users()) {
4226 if (isa<SelectInst>(U)) {
4227 assert(!Sel && "Reduction exit feeding two selects");
4228 Sel = U;
4229 } else
4230 assert(isa<PHINode>(U) && "Reduction exit must feed Phi's or select");
4231 }
4232 assert(Sel && "Reduction exit feeds no select");
4233 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, Sel);
4234
4235 // If the target can create a predicated operator for the reduction at no
4236 // extra cost in the loop (for example a predicated vadd), it can be
4237 // cheaper for the select to remain in the loop than be sunk out of it,
4238 // and so use the select value for the phi instead of the old
4239 // LoopExitValue.
4240 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];
4241 if (PreferPredicatedReductionSelect ||
4242 TTI->preferPredicatedReductionSelect(
4243 RdxDesc.getOpcode(), Phi->getType(),
4244 TargetTransformInfo::ReductionFlags())) {
4245 auto *VecRdxPhi = cast<PHINode>(getOrCreateVectorValue(Phi, Part));
4246 VecRdxPhi->setIncomingValueForBlock(
4247 LI->getLoopFor(LoopVectorBody)->getLoopLatch(), Sel);
4248 }
4249 }
4250 }
4251
4252 // If the vector reduction can be performed in a smaller type, we truncate
4253 // then extend the loop exit value to enable InstCombine to evaluate the
4254 // entire expression in the smaller type.
4255 if (VF.isVector() && Phi->getType() != RdxDesc.getRecurrenceType()) {
4256 assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!");
4257 assert(!VF.isScalable() && "scalable vectors not yet supported.");
4258 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
4259 Builder.SetInsertPoint(
4260 LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
4261 VectorParts RdxParts(UF);
4262 for (unsigned Part = 0; Part < UF; ++Part) {
4263 RdxParts[Part] = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
4264 Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
4265 Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
4266 : Builder.CreateZExt(Trunc, VecTy);
4267 for (Value::user_iterator UI = RdxParts[Part]->user_begin();
4268 UI != RdxParts[Part]->user_end();)
4269 if (*UI != Trunc) {
4270 (*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd);
4271 RdxParts[Part] = Extnd;
4272 } else {
4273 ++UI;
4274 }
4275 }
4276 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
4277 for (unsigned Part = 0; Part < UF; ++Part) {
4278 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
4279 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, RdxParts[Part]);
4280 }
4281 }
4282
4283 // Reduce all of the unrolled parts into a single vector.
4284 Value *ReducedPartRdx = VectorLoopValueMap.getVectorValue(LoopExitInst, 0);
4285 unsigned Op = RecurrenceDescriptor::getOpcode(RK);
4286
4287 // The middle block terminator has already been assigned a DebugLoc here (the
4288 // OrigLoop's single latch terminator). We want the whole middle block to
4289 // appear to execute on this line because: (a) it is all compiler generated,
4290 // (b) these instructions are always executed after evaluating the latch
4291 // conditional branch, and (c) other passes may add new predecessors which
4292 // terminate on this line. This is the easiest way to ensure we don't
4293 // accidentally cause an extra step back into the loop while debugging.
4294 setDebugLocFromInst(Builder, LoopMiddleBlock->getTerminator());
4295 for (unsigned Part = 1; Part < UF; ++Part) {
4296 Value *RdxPart = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
4297 if (Op != Instruction::ICmp && Op != Instruction::FCmp)
4298 // Floating point operations had to be 'fast' to enable the reduction.
4299 ReducedPartRdx = addFastMathFlag(
4300 Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxPart,
4301 ReducedPartRdx, "bin.rdx"),
4302 RdxDesc.getFastMathFlags());
4303 else
4304 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
4305 }
4306
4307 // Create the reduction after the loop. Note that inloop reductions create the
4308 // target reduction in the loop using a Reduction recipe.
4309 if (VF.isVector() && !IsInLoopReductionPhi) {
4310 ReducedPartRdx =
4311 createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx);
4312 // If the reduction can be performed in a smaller type, we need to extend
4313 // the reduction to the wider type before we branch to the original loop.
4314 if (Phi->getType() != RdxDesc.getRecurrenceType())
4315 ReducedPartRdx =
4316 RdxDesc.isSigned()
4317 ? Builder.CreateSExt(ReducedPartRdx, Phi->getType())
4318 : Builder.CreateZExt(ReducedPartRdx, Phi->getType());
4319 }
4320
4321 // Create a phi node that merges control-flow from the backedge-taken check
4322 // block and the middle block.
4323 PHINode *BCBlockPhi = PHINode::Create(Phi->getType(), 2, "bc.merge.rdx",
4324 LoopScalarPreHeader->getTerminator());
4325 for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
4326 BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);
4327 BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);
4328
4329 // Now, we need to fix the users of the reduction variable
4330 // inside and outside of the scalar remainder loop.
4331
4332 // We know that the loop is in LCSSA form. We need to update the PHI nodes
4333 // in the exit blocks. See comment on analogous loop in
4334 // fixFirstOrderRecurrence for a more complete explaination of the logic.
4335 for (PHINode &LCSSAPhi : LoopExitBlock->phis())
4336 if (any_of(LCSSAPhi.incoming_values(),
4337 [LoopExitInst](Value *V) { return V == LoopExitInst; }))
4338 LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock);
4339
4340 // Fix the scalar loop reduction variable with the incoming reduction sum
4341 // from the vector body and from the backedge value.
4342 int IncomingEdgeBlockIdx =
4343 Phi->getBasicBlockIndex(OrigLoop->getLoopLatch());
4344 assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index");
4345 // Pick the other block.
4346 int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
4347 Phi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
4348 Phi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
4349}
4350
4351void InnerLoopVectorizer::clearReductionWrapFlags(
4352 RecurrenceDescriptor &RdxDesc) {
4353 RecurKind RK = RdxDesc.getRecurrenceKind();
4354 if (RK != RecurKind::Add && RK != RecurKind::Mul)
4355 return;
4356
4357 Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr();
4358 assert(LoopExitInstr && "null loop exit instruction");
4359 SmallVector<Instruction *, 8> Worklist;
4360 SmallPtrSet<Instruction *, 8> Visited;
4361 Worklist.push_back(LoopExitInstr);
4362 Visited.insert(LoopExitInstr);
4363
4364 while (!Worklist.empty()) {
4365 Instruction *Cur = Worklist.pop_back_val();
4366 if (isa<OverflowingBinaryOperator>(Cur))
4367 for (unsigned Part = 0; Part < UF; ++Part) {
4368 Value *V = getOrCreateVectorValue(Cur, Part);
4369 cast<Instruction>(V)->dropPoisonGeneratingFlags();
4370 }
4371
4372 for (User *U : Cur->users()) {
4373 Instruction *UI = cast<Instruction>(U);
4374 if ((Cur != LoopExitInstr || OrigLoop->contains(UI->getParent())) &&
4375 Visited.insert(UI).second)
4376 Worklist.push_back(UI);
4377 }
4378 }
4379}
4380
4381void InnerLoopVectorizer::fixLCSSAPHIs() {
4382 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
4383 if (LCSSAPhi.getBasicBlockIndex(LoopMiddleBlock) != -1)
4384 // Some phis were already hand updated by the reduction and recurrence
4385 // code above, leave them alone.
4386 continue;
4387
4388 auto *IncomingValue = LCSSAPhi.getIncomingValue(0);
4389 // Non-instruction incoming values will have only one value.
4390 unsigned LastLane = 0;
4391 if (isa<Instruction>(IncomingValue))
4392 LastLane = Cost->isUniformAfterVectorization(
4393 cast<Instruction>(IncomingValue), VF)
4394 ? 0
4395 : VF.getKnownMinValue() - 1;
4396 assert((!VF.isScalable() || LastLane == 0) &&
4397 "scalable vectors dont support non-uniform scalars yet");
4398 // Can be a loop invariant incoming value or the last scalar value to be
4399 // extracted from the vectorized loop.
4400 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
4401 Value *lastIncomingValue =
4402 getOrCreateScalarValue(IncomingValue, { UF - 1, LastLane });
4403 LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock);
4404 }
4405}
4406
4407void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
4408 // The basic block and loop containing the predicated instruction.
4409 auto *PredBB = PredInst->getParent();
4410 auto *VectorLoop = LI->getLoopFor(PredBB);
4411
4412 // Initialize a worklist with the operands of the predicated instruction.
4413 SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());
4414
4415 // Holds instructions that we need to analyze again. An instruction may be
4416 // reanalyzed if we don't yet know if we can sink it or not.
4417 SmallVector<Instruction *, 8> InstsToReanalyze;
4418
4419 // Returns true if a given use occurs in the predicated block. Phi nodes use
4420 // their operands in their corresponding predecessor blocks.
4421 auto isBlockOfUsePredicated = [&](Use &U) -> bool {
4422 auto *I = cast<Instruction>(U.getUser());
4423 BasicBlock *BB = I->getParent();
4424 if (auto *Phi = dyn_cast<PHINode>(I))
4425 BB = Phi->getIncomingBlock(
4426 PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
4427 return BB == PredBB;
4428 };
4429
4430 // Iteratively sink the scalarized operands of the predicated instruction
4431 // into the block we created for it. When an instruction is sunk, it's
4432 // operands are then added to the worklist. The algorithm ends after one pass
4433 // through the worklist doesn't sink a single instruction.
4434 bool Changed;
4435 do {
4436 // Add the instructions that need to be reanalyzed to the worklist, and
4437 // reset the changed indicator.
4438 Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end());
4439 InstsToReanalyze.clear();
4440 Changed = false;
4441
4442 while (!Worklist.empty()) {
4443 auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());
4444
4445 // We can't sink an instruction if it is a phi node, is already in the
4446 // predicated block, is not in the loop, or may have side effects.
4447 if (!I || isa<PHINode>(I) || I->getParent() == PredBB ||
4448 !VectorLoop->contains(I) || I->mayHaveSideEffects())
4449 continue;
4450
4451 // It's legal to sink the instruction if all its uses occur in the
4452 // predicated block. Otherwise, there's nothing to do yet, and we may
4453 // need to reanalyze the instruction.
4454 if (!llvm::all_of(I->uses(), isBlockOfUsePredicated)) {
4455 InstsToReanalyze.push_back(I);
4456 continue;
4457 }
4458
4459 // Move the instruction to the beginning of the predicated block, and add
4460 // it's operands to the worklist.
4461 I->moveBefore(&*PredBB->getFirstInsertionPt());
4462 Worklist.insert(I->op_begin(), I->op_end());
4463
4464 // The sinking may have enabled other instructions to be sunk, so we will
4465 // need to iterate.
4466 Changed = true;
4467 }
4468 } while (Changed);
4469}
4470
4471void InnerLoopVectorizer::fixNonInductionPHIs() {
4472 for (PHINode *OrigPhi : OrigPHIsToFix) {
4473 PHINode *NewPhi =
4474 cast<PHINode>(VectorLoopValueMap.getVectorValue(OrigPhi, 0));
4475 unsigned NumIncomingValues = OrigPhi->getNumIncomingValues();
4476
4477 SmallVector<BasicBlock *, 2> ScalarBBPredecessors(
4478 predecessors(OrigPhi->getParent()));
4479 SmallVector<BasicBlock *, 2> VectorBBPredecessors(
4480 predecessors(NewPhi->getParent()));
4481 assert(ScalarBBPredecessors.size() == VectorBBPredecessors.size() &&
4482 "Scalar and Vector BB should have the same number of predecessors");
4483
4484 // The insertion point in Builder may be invalidated by the time we get
4485 // here. Force the Builder insertion point to something valid so that we do
4486 // not run into issues during insertion point restore in
4487 // getOrCreateVectorValue calls below.
4488 Builder.SetInsertPoint(NewPhi);
4489
4490 // The predecessor order is preserved and we can rely on mapping between
4491 // scalar and vector block predecessors.
4492 for (unsigned i = 0; i < NumIncomingValues; ++i) {
4493 BasicBlock *NewPredBB = VectorBBPredecessors[i];
4494
4495 // When looking up the new scalar/vector values to fix up, use incoming
4496 // values from original phi.
4497 Value *ScIncV =
4498 OrigPhi->getIncomingValueForBlock(ScalarBBPredecessors[i]);
4499
4500 // Scalar incoming value may need a broadcast
4501 Value *NewIncV = getOrCreateVectorValue(ScIncV, 0);
4502 NewPhi->addIncoming(NewIncV, NewPredBB);
4503 }
4504 }
4505}
4506
4507void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef,
4508 VPUser &Operands, unsigned UF,
4509 ElementCount VF, bool IsPtrLoopInvariant,
4510 SmallBitVector &IsIndexLoopInvariant,
4511 VPTransformState &State) {
4512 // Construct a vector GEP by widening the operands of the scalar GEP as
4513 // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
4514 // results in a vector of pointers when at least one operand of the GEP
4515 // is vector-typed. Thus, to keep the representation compact, we only use
4516 // vector-typed operands for loop-varying values.
4517
4518 if (VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
4519 // If we are vectorizing, but the GEP has only loop-invariant operands,
4520 // the GEP we build (by only using vector-typed operands for
4521 // loop-varying values) would be a scalar pointer. Thus, to ensure we
4522 // produce a vector of pointers, we need to either arbitrarily pick an
4523 // operand to broadcast, or broadcast a clone of the original GEP.
4524 // Here, we broadcast a clone of the original.
4525 //
4526 // TODO: If at some point we decide to scalarize instructions having
4527 // loop-invariant operands, this special case will no longer be
4528 // required. We would add the scalarization decision to
4529 // collectLoopScalars() and teach getVectorValue() to broadcast
4530 // the lane-zero scalar value.
4531 auto *Clone = Builder.Insert(GEP->clone());
4532 for (unsigned Part = 0; Part < UF; ++Part) {
4533 Value *EntryPart = Builder.CreateVectorSplat(VF, Clone);
4534 State.set(VPDef, GEP, EntryPart, Part);
4535 addMetadata(EntryPart, GEP);
4536 }
4537 } else {
4538 // If the GEP has at least one loop-varying operand, we are sure to
4539 // produce a vector of pointers. But if we are only unrolling, we want
4540 // to produce a scalar GEP for each unroll part. Thus, the GEP we
4541 // produce with the code below will be scalar (if VF == 1) or vector
4542 // (otherwise). Note that for the unroll-only case, we still maintain
4543 // values in the vector mapping with initVector, as we do for other
4544 // instructions.
4545 for (unsigned Part = 0; Part < UF; ++Part) {
4546 // The pointer operand of the new GEP. If it's loop-invariant, we
4547 // won't broadcast it.
4548 auto *Ptr = IsPtrLoopInvariant ? State.get(Operands.getOperand(0), {0, 0})
4549 : State.get(Operands.getOperand(0), Part);
4550
4551 // Collect all the indices for the new GEP. If any index is
4552 // loop-invariant, we won't broadcast it.
4553 SmallVector<Value *, 4> Indices;
4554 for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) {
4555 VPValue *Operand = Operands.getOperand(I);
4556 if (IsIndexLoopInvariant[I - 1])
4557 Indices.push_back(State.get(Operand, {0, 0}));
4558 else
4559 Indices.push_back(State.get(Operand, Part));
4560 }
4561
4562 // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
4563 // but it should be a vector, otherwise.
4564 auto *NewGEP =
4565 GEP->isInBounds()
4566 ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr,
4567 Indices)
4568 : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices);
4569 assert((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
4570 "NewGEP is not a pointer vector");
4571 State.set(VPDef, GEP, NewGEP, Part);
4572 addMetadata(NewGEP, GEP);
4573 }
4574 }
4575}
4576
4577void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
4578 RecurrenceDescriptor *RdxDesc,
4579 Value *StartV, unsigned UF,
4580 ElementCount VF) {
4581 assert(!VF.isScalable() && "scalable vectors not yet supported.");
4582 PHINode *P = cast<PHINode>(PN);
4583 if (EnableVPlanNativePath) {
4584 // Currently we enter here in the VPlan-native path for non-induction
4585 // PHIs where all control flow is uniform. We simply widen these PHIs.
4586 // Create a vector phi with no operands - the vector phi operands will be
4587 // set at the end of vector code generation.
4588 Type *VecTy =
4589 (VF.isScalar()) ? PN->getType() : VectorType::get(PN->getType(), VF);
4590 Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi");
4591 VectorLoopValueMap.setVectorValue(P, 0, VecPhi);
4592 OrigPHIsToFix.push_back(P);
4593
4594 return;
4595 }
4596
4597 assert(PN->getParent() == OrigLoop->getHeader() &&
4598 "Non-header phis should have been handled elsewhere");
4599
4600 // In order to support recurrences we need to be able to vectorize Phi nodes.
4601 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
4602 // stage #1: We create a new vector PHI node with no incoming edges. We'll use
4603 // this value when we vectorize all of the instructions that use the PHI.
4604 if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
4605 Value *Iden = nullptr;
4606 bool ScalarPHI =
4607 (VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
4608 Type *VecTy =
4609 ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), VF);
4610
4611 if (RdxDesc) {
4612 assert(Legal->isReductionVariable(P) && StartV &&
4613 "RdxDesc should only be set for reduction variables; in that case "
4614 "a StartV is also required");
4615 RecurKind RK = RdxDesc->getRecurrenceKind();
4616 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
4617 // MinMax reduction have the start value as their identify.
4618 if (ScalarPHI) {
4619 Iden = StartV;
4620 } else {
4621 IRBuilderBase::InsertPointGuard IPBuilder(Builder);
4622 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
4623 StartV = Iden = Builder.CreateVectorSplat(VF, StartV, "minmax.ident");
4624 }
4625 } else {
4626 Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
4627 RK, VecTy->getScalarType());
4628 Iden = IdenC;
4629
4630 if (!ScalarPHI) {
4631 Iden = ConstantVector::getSplat(VF, IdenC);
4632 IRBuilderBase::InsertPointGuard IPBuilder(Builder);
4633 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
4634 Constant *Zero = Builder.getInt32(0);
4635 StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
4636 }
4637 }
4638 }
4639
4640 for (unsigned Part = 0; Part < UF; ++Part) {
4641 // This is phase one of vectorizing PHIs.
4642 Value *EntryPart = PHINode::Create(
4643 VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
4644 VectorLoopValueMap.setVectorValue(P, Part, EntryPart);
4645 if (StartV) {
4646 // Make sure to add the reduction start value only to the
4647 // first unroll part.
4648 Value *StartVal = (Part == 0) ? StartV : Iden;
4649 cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader);
4650 }
4651 }
4652 return;
4653 }
4654
4655 assert(!Legal->isReductionVariable(P) &&
4656 "reductions should be handled above");
4657
4658 setDebugLocFromInst(Builder, P);
4659
4660 // This PHINode must be an induction variable.
4661 // Make sure that we know about it.
4662 assert(Legal->getInductionVars().count(P) && "Not an induction variable");
4663
4664 InductionDescriptor II = Legal->getInductionVars().lookup(P);
4665 const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
4666
4667 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
4668 // which can be found from the original scalar operations.
4669 switch (II.getKind()) {
4670 case InductionDescriptor::IK_NoInduction:
4671 llvm_unreachable("Unknown induction");
4672 case InductionDescriptor::IK_IntInduction:
4673 case InductionDescriptor::IK_FpInduction:
4674 llvm_unreachable("Integer/fp induction is handled elsewhere.");
4675 case InductionDescriptor::IK_PtrInduction: {
4676 // Handle the pointer induction variable case.
4677 assert(P->getType()->isPointerTy() && "Unexpected type.");
4678
4679 if (Cost->isScalarAfterVectorization(P, VF)) {
4680 // This is the normalized GEP that starts counting at zero.
4681 Value *PtrInd =
4682 Builder.CreateSExtOrTrunc(Induction, II.getStep()->getType());
4683 // Determine the number of scalars we need to generate for each unroll
4684 // iteration. If the instruction is uniform, we only need to generate the
4685 // first lane. Otherwise, we generate all VF values.
4686 unsigned Lanes =
4687 Cost->isUniformAfterVectorization(P, VF) ? 1 : VF.getKnownMinValue();
4688 for (unsigned Part = 0; Part < UF; ++Part) {
4689 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
4690 Constant *Idx = ConstantInt::get(PtrInd->getType(),
4691 Lane + Part * VF.getKnownMinValue());
4692 Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
4693 Value *SclrGep =
4694 emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II);
4695 SclrGep->setName("next.gep");
4696 VectorLoopValueMap.setScalarValue(P, {Part, Lane}, SclrGep);
4697 }
4698 }
4699 return;
4700 }
4701 assert(isa<SCEVConstant>(II.getStep()) &&
4702 "Induction step not a SCEV constant!");
4703 Type *PhiType = II.getStep()->getType();
4704
4705 // Build a pointer phi
4706 Value *ScalarStartValue = II.getStartValue();
4707 Type *ScStValueType = ScalarStartValue->getType();
4708 PHINode *NewPointerPhi =
4709 PHINode::Create(ScStValueType, 2, "pointer.phi", Induction);
4710 NewPointerPhi->addIncoming(ScalarStartValue, LoopVectorPreHeader);
4711
4712 // A pointer induction, performed by using a gep
4713 BasicBlock *LoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
4714 Instruction *InductionLoc = LoopLatch->getTerminator();
4715 const SCEV *ScalarStep = II.getStep();
4716 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
4717 Value *ScalarStepValue =
4718 Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc);
4719 Value *InductionGEP = GetElementPtrInst::Create(
4720 ScStValueType->getPointerElementType(), NewPointerPhi,
4721 Builder.CreateMul(
4722 ScalarStepValue,
4723 ConstantInt::get(PhiType, VF.getKnownMinValue() * UF)),
4724 "ptr.ind", InductionLoc);
4725 NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
4726
4727 // Create UF many actual address geps that use the pointer
4728 // phi as base and a vectorized version of the step value
4729 // (<step*0, ..., step*N>) as offset.
4730 for (unsigned Part = 0; Part < UF; ++Part) {
4731 SmallVector<Constant *, 8> Indices;
4732 // Create a vector of consecutive numbers from zero to VF.
4733 for (unsigned i = 0; i < VF.getKnownMinValue(); ++i)
4734 Indices.push_back(
4735 ConstantInt::get(PhiType, i + Part * VF.getKnownMinValue()));
4736 Constant *StartOffset = ConstantVector::get(Indices);
4737
4738 Value *GEP = Builder.CreateGEP(
4739 ScStValueType->getPointerElementType(), NewPointerPhi,
4740 Builder.CreateMul(
4741 StartOffset,
4742 Builder.CreateVectorSplat(VF.getKnownMinValue(), ScalarStepValue),
4743 "vector.gep"));
4744 VectorLoopValueMap.setVectorValue(P, Part, GEP);
4745 }
4746 }
4747 }
4748}
4749
4750/// A helper function for checking whether an integer division-related
4751/// instruction may divide by zero (in which case it must be predicated if
4752/// executed conditionally in the scalar code).
4753/// TODO: It may be worthwhile to generalize and check isKnownNonZero().
4754/// Non-zero divisors that are non compile-time constants will not be
4755/// converted into multiplication, so we will still end up scalarizing
4756/// the division, but can do so w/o predication.
4757static bool mayDivideByZero(Instruction &I) {
4758 assert((I.getOpcode() == Instruction::UDiv ||
4759 I.getOpcode() == Instruction::SDiv ||
4760 I.getOpcode() == Instruction::URem ||
4761 I.getOpcode() == Instruction::SRem) &&
4762 "Unexpected instruction");
4763 Value *Divisor = I.getOperand(1);
4764 auto *CInt = dyn_cast<ConstantInt>(Divisor);
4765 return !CInt || CInt->isZero();
4766}
4767
4768void InnerLoopVectorizer::widenInstruction(Instruction &I, VPValue *Def,
4769 VPUser &User,
4770 VPTransformState &State) {
4771 switch (I.getOpcode()) {
4772 case Instruction::Call:
4773 case Instruction::Br:
4774 case Instruction::PHI:
4775 case Instruction::GetElementPtr:
4776 case Instruction::Select:
4777 llvm_unreachable("This instruction is handled by a different recipe.");
4778 case Instruction::UDiv:
4779 case Instruction::SDiv:
4780 case Instruction::SRem:
4781 case Instruction::URem:
4782 case Instruction::Add:
4783 case Instruction::FAdd:
4784 case Instruction::Sub:
4785 case Instruction::FSub:
4786 case Instruction::FNeg:
4787 case Instruction::Mul:
4788 case Instruction::FMul:
4789 case Instruction::FDiv:
4790 case Instruction::FRem:
4791 case Instruction::Shl:
4792 case Instruction::LShr:
4793 case Instruction::AShr:
4794 case Instruction::And:
4795 case Instruction::Or:
4796 case Instruction::Xor: {
4797 // Just widen unops and binops.
4798 setDebugLocFromInst(Builder, &I);
4799
4800 for (unsigned Part = 0; Part < UF; ++Part) {
4801 SmallVector<Value *, 2> Ops;
4802 for (VPValue *VPOp : User.operands())
4803 Ops.push_back(State.get(VPOp, Part));
4804
4805 Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
4806
4807 if (auto *VecOp = dyn_cast<Instruction>(V))
4808 VecOp->copyIRFlags(&I);
4809
4810 // Use this vector value for all users of the original instruction.
4811 State.set(Def, &I, V, Part);
4812 addMetadata(V, &I);
4813 }
4814
4815 break;
4816 }
4817 case Instruction::ICmp:
4818 case Instruction::FCmp: {
4819 // Widen compares. Generate vector compares.
4820 bool FCmp = (I.getOpcode() == Instruction::FCmp);
4821 auto *Cmp = cast<CmpInst>(&I);
4822 setDebugLocFromInst(Builder, Cmp);
4823 for (unsigned Part = 0; Part < UF; ++Part) {
4824 Value *A = State.get(User.getOperand(0), Part);
4825 Value *B = State.get(User.getOperand(1), Part);
4826 Value *C = nullptr;
4827 if (FCmp) {
4828 // Propagate fast math flags.
4829 IRBuilder<>::FastMathFlagGuard FMFG(Builder);
4830 Builder.setFastMathFlags(Cmp->getFastMathFlags());
4831 C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
4832 } else {
4833 C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
4834 }
4835 State.set(Def, &I, C, Part);
4836 addMetadata(C, &I);
4837 }
4838
4839 break;
4840 }
4841
4842 case Instruction::ZExt:
4843 case Instruction::SExt:
4844 case Instruction::FPToUI:
4845 case Instruction::FPToSI:
4846 case Instruction::FPExt:
4847 case Instruction::PtrToInt:
4848 case Instruction::IntToPtr:
4849 case Instruction::SIToFP:
4850 case Instruction::UIToFP:
4851 case Instruction::Trunc:
4852 case Instruction::FPTrunc:
4853 case Instruction::BitCast: {
4854 auto *CI = cast<CastInst>(&I);
4855 setDebugLocFromInst(Builder, CI);
4856
4857 /// Vectorize casts.
4858 Type *DestTy =
4859 (VF.isScalar()) ? CI->getType() : VectorType::get(CI->getType(), VF);
4860
4861 for (unsigned Part = 0; Part < UF; ++Part) {
4862 Value *A = State.get(User.getOperand(0), Part);
4863 Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
4864 State.set(Def, &I, Cast, Part);
4865 addMetadata(Cast, &I);
4866 }
4867 break;
4868 }
4869 default:
4870 // This instruction is not vectorized by simple widening.
4871 LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
4872 llvm_unreachable("Unhandled instruction!");
4873 } // end of switch.
4874}
4875
4876void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
4877 VPUser &ArgOperands,
4878 VPTransformState &State) {
4879 assert(!isa<DbgInfoIntrinsic>(I) &&
4880 "DbgInfoIntrinsic should have been dropped during VPlan construction");
4881 setDebugLocFromInst(Builder, &I);
4882
4883 Module *M = I.getParent()->getParent()->getParent();
4884 auto *CI = cast<CallInst>(&I);
4885
4886 SmallVector<Type *, 4> Tys;
4887 for (Value *ArgOperand : CI->arg_operands())
4888 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));
4889
4890 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
4891
4892 // The flag shows whether we use Intrinsic or a usual Call for vectorized
4893 // version of the instruction.
4894 // Is it beneficial to perform intrinsic call compared to lib call?
4895 bool NeedToScalarize = false;
4896 InstructionCost CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize);
4897 InstructionCost IntrinsicCost = ID ? Cost->getVectorIntrinsicCost(CI, VF) : 0;
4898 bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost;
4899 assert((UseVectorIntrinsic || !NeedToScalarize) &&
4900 "Instruction should be scalarized elsewhere.");
4901 assert(IntrinsicCost.isValid() && CallCost.isValid() &&
4902 "Cannot have invalid costs while widening");
4903
4904 for (unsigned Part = 0; Part < UF; ++Part) {
4905 SmallVector<Value *, 4> Args;
4906 for (auto &I : enumerate(ArgOperands.operands())) {
4907 // Some intrinsics have a scalar argument - don't replace it with a
4908 // vector.
4909 Value *Arg;
4910 if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
4911 Arg = State.get(I.value(), Part);
4912 else
4913 Arg = State.get(I.value(), {0, 0});
4914 Args.push_back(Arg);
4915 }
4916
4917 Function *VectorF;
4918 if (UseVectorIntrinsic) {
4919 // Use vector version of the intrinsic.
4920 Type *TysForDecl[] = {CI->getType()};
4921 if (VF.isVector()) {
4922 assert(!VF.isScalable() && "VF is assumed to be non scalable.");
4923 TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
4924 }
4925 VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
4926 assert(VectorF && "Can't retrieve vector intrinsic.");
4927 } else {
4928 // Use vector version of the function call.
4929 const VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
4930#ifndef NDEBUG
4931 assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
4932 "Can't create vector function.");
4933#endif
4934 VectorF = VFDatabase(*CI).getVectorizedFunction(Shape);
4935 }
4936 SmallVector<OperandBundleDef, 1> OpBundles;
4937 CI->getOperandBundlesAsDefs(OpBundles);
4938 CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
4939
4940 if (isa<FPMathOperator>(V))
4941 V->copyFastMathFlags(CI);
4942
4943 State.set(Def, &I, V, Part);
4944 addMetadata(V, &I);
4945 }
4946}
4947
4948void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef,
4949 VPUser &Operands,
4950 bool InvariantCond,
4951 VPTransformState &State) {
4952 setDebugLocFromInst(Builder, &I);
4953
4954 // The condition can be loop invariant but still defined inside the
4955 // loop. This means that we can't just use the original 'cond' value.
4956 // We have to take the 'vectorized' value and pick the first lane.
4957 // Instcombine will make this a no-op.
4958 auto *InvarCond =
4959 InvariantCond ? State.get(Operands.getOperand(0), {0, 0}) : nullptr;
4960
4961 for (unsigned Part = 0; Part < UF; ++Part) {
4962 Value *Cond =
4963 InvarCond ? InvarCond : State.get(Operands.getOperand(0), Part);
4964 Value *Op0 = State.get(Operands.getOperand(1), Part);
4965 Value *Op1 = State.get(Operands.getOperand(2), Part);
4966 Value *Sel = Builder.CreateSelect(Cond, Op0, Op1);
4967 State.set(VPDef, &I, Sel, Part);
4968 addMetadata(Sel, &I);
4969 }
4970}
4971
4972void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
4973 // We should not collect Scalars more than once per VF. Right now, this
4974 // function is called from collectUniformsAndScalars(), which already does
4975 // this check. Collecting Scalars for VF=1 does not make any sense.
4976 assert(VF.isVector() && Scalars.find(VF) == Scalars.end() &&
4977 "This function should not be visited twice for the same VF");
4978
4979 SmallSetVector<Instruction *, 8> Worklist;
4980
4981 // These sets are used to seed the analysis with pointers used by memory
4982 // accesses that will remain scalar.
4983 SmallSetVector<Instruction *, 8> ScalarPtrs;
4984 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
4985 auto *Latch = TheLoop->getLoopLatch();
4986
4987 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
4988 // The pointer operands of loads and stores will be scalar as long as the
4989 // memory access is not a gather or scatter operation. The value operand of a
4990 // store will remain scalar if the store is scalarized.
4991 auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
4992 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
4993 assert(WideningDecision != CM_Unknown &&
4994 "Widening decision should be ready at this moment");
4995 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
4996 if (Ptr == Store->getValueOperand())
4997 return WideningDecision == CM_Scalarize;
4998 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
4999 "Ptr is neither a value or pointer operand");
5000 return WideningDecision != CM_GatherScatter;
5001 };
5002
5003 // A helper that returns true if the given value is a bitcast or
5004 // getelementptr instruction contained in the loop.
5005 auto isLoopVaryingBitCastOrGEP = [&](Value *V) {
5006 return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) ||
5007 isa<GetElementPtrInst>(V)) &&
5008 !TheLoop->isLoopInvariant(V);
5009 };
5010
5011 auto isScalarPtrInduction = [&](Instruction *MemAccess, Value *Ptr) {
5012 if (!isa<PHINode>(Ptr) ||
5013 !Legal->getInductionVars().count(cast<PHINode>(Ptr)))
5014 return false;
5015 auto &Induction = Legal->getInductionVars()[cast<PHINode>(Ptr)];
5016 if (Induction.getKind() != InductionDescriptor::IK_PtrInduction)
5017 return false;
5018 return isScalarUse(MemAccess, Ptr);
5019 };
5020
5021 // A helper that evaluates a memory access's use of a pointer. If the
5022 // pointer is actually the pointer induction of a loop, it is being
5023 // inserted into Worklist. If the use will be a scalar use, and the
5024 // pointer is only used by memory accesses, we place the pointer in
5025 // ScalarPtrs. Otherwise, the pointer is placed in PossibleNonScalarPtrs.
5026 auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
5027 if (isScalarPtrInduction(MemAccess, Ptr)) {
5028 Worklist.insert(cast<Instruction>(Ptr));
5029 Instruction *Update = cast<Instruction>(
5030 cast<PHINode>(Ptr)->getIncomingValueForBlock(Latch));
5031 Worklist.insert(Update);
5032 LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Ptr
5033 << "\n");
5034 LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Update
5035 << "\n");
5036 return;
5037 }
5038 // We only care about bitcast and getelementptr instructions contained in
5039 // the loop.
5040 if (!isLoopVaryingBitCastOrGEP(Ptr))
5041 return;
5042
5043 // If the pointer has already been identified as scalar (e.g., if it was
5044 // also identified as uniform), there's nothing to do.
5045 auto *I = cast<Instruction>(Ptr);
5046 if (Worklist.count(I))
5047 return;
5048
5049 // If the use of the pointer will be a scalar use, and all users of the
5050 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
5051 // place the pointer in PossibleNonScalarPtrs.
5052 if (isScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) {
5053 return isa<LoadInst>(U) || isa<StoreInst>(U);
5054 }))
5055 ScalarPtrs.insert(I);
5056 else
5057 PossibleNonScalarPtrs.insert(I);
5058 };
5059
5060 // We seed the scalars analysis with three classes of instructions: (1)
5061 // instructions marked uniform-after-vectorization and (2) bitcast,
5062 // getelementptr and (pointer) phi instructions used by memory accesses
5063 // requiring a scalar use.
5064 //
5065 // (1) Add to the worklist all instructions that have been identified as
5066 // uniform-after-vectorization.
5067 Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end());
5068
5069 // (2) Add to the worklist all bitcast and getelementptr instructions used by
5070 // memory accesses requiring a scalar use. The pointer operands of loads and
5071 // stores will be scalar as long as the memory accesses is not a gather or
5072 // scatter operation. The value operand of a store will remain scalar if the
5073 // store is scalarized.
5074 for (auto *BB : TheLoop->blocks())
5075 for (auto &I : *BB) {
5076 if (auto *Load = dyn_cast<LoadInst>(&I)) {
5077 evaluatePtrUse(Load, Load->getPointerOperand());
5078 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
5079 evaluatePtrUse(Store, Store->getPointerOperand());
5080 evaluatePtrUse(Store, Store->getValueOperand());
5081 }
5082 }
5083 for (auto *I : ScalarPtrs)
5084 if (!PossibleNonScalarPtrs.count(I)) {
5085 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
5086 Worklist.insert(I);
5087 }
5088
5089 // Insert the forced scalars.
5090 // FIXME: Currently widenPHIInstruction() often creates a dead vector
5091 // induction variable when the PHI user is scalarized.
5092 auto ForcedScalar = ForcedScalars.find(VF);
5093 if (ForcedScalar != ForcedScalars.end())
5094 for (auto *I : ForcedScalar->second)
5095 Worklist.insert(I);
5096
5097 // Expand the worklist by looking through any bitcasts and getelementptr
5098 // instructions we've already identified as scalar. This is similar to the
5099 // expansion step in collectLoopUniforms(); however, here we're only
5100 // expanding to include additional bitcasts and getelementptr instructions.
5101 unsigned Idx = 0;
5102 while (Idx != Worklist.size()) {
5103 Instruction *Dst = Worklist[Idx++];
5104 if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))
5105 continue;
5106 auto *Src = cast<Instruction>(Dst->getOperand(0));
5107 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
5108 auto *J = cast<Instruction>(U);
5109 return !TheLoop->contains(J) || Worklist.count(J) ||
5110 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
5111 isScalarUse(J, Src));
5112 })) {
5113 Worklist.insert(Src);
5114 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
5115 }
5116 }
5117
5118 // An induction variable will remain scalar if all users of the induction
5119 // variable and induction variable update remain scalar.
5120 for (auto &Induction : Legal->getInductionVars()) {
5121 auto *Ind = Induction.first;
5122 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
5123
5124 // If tail-folding is applied, the primary induction variable will be used
5125 // to feed a vector compare.
5126 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
5127 continue;
5128
5129 // Determine if all users of the induction variable are scalar after
5130 // vectorization.
5131 auto ScalarInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
5132 auto *I = cast<Instruction>(U);
5133 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I);
5134 });
5135 if (!ScalarInd)
5136 continue;
5137
5138 // Determine if all users of the induction variable update instruction are
5139 // scalar after vectorization.
5140 auto ScalarIndUpdate =
5141 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
5142 auto *I = cast<Instruction>(U);
5143 return I == Ind || !TheLoop->contains(I) || Worklist.count(I);
5144 });
5145 if (!ScalarIndUpdate)
5146 continue;
5147
5148 // The induction variable and its update instruction will remain scalar.
5149 Worklist.insert(Ind);
5150 Worklist.insert(IndUpdate);
5151 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
5152 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
5153 << "\n");
5154 }
5155
5156 Scalars[VF].insert(Worklist.begin(), Worklist.end());
5157}
5158
5159bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
5160 ElementCount VF) {
5161 if (!blockNeedsPredication(I->getParent()))
5162 return false;
5163 switch(I->getOpcode()) {
5164 default:
5165 break;
5166 case Instruction::Load:
5167 case Instruction::Store: {
5168 if (!Legal->isMaskRequired(I))
5169 return false;
5170 auto *Ptr = getLoadStorePointerOperand(I);
5171 auto *Ty = getMemInstValueType(I);
5172 // We have already decided how to vectorize this instruction, get that
5173 // result.
5174 if (VF.isVector()) {
5175 InstWidening WideningDecision = getWideningDecision(I, VF);
5176 assert(WideningDecision != CM_Unknown &&
5177 "Widening decision should be ready at this moment");
5178 return WideningDecision == CM_Scalarize;
5179 }
5180 const Align Alignment = getLoadStoreAlignment(I);
5181 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
5182 isLegalMaskedGather(Ty, Alignment))
5183 : !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
5184 isLegalMaskedScatter(Ty, Alignment));
5185 }
5186 case Instruction::UDiv:
5187 case Instruction::SDiv:
5188 case Instruction::SRem:
5189 case Instruction::URem:
5190 return mayDivideByZero(*I);
5191 }
5192 return false;
5193}
5194
5195bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
5196 Instruction *I, ElementCount VF) {
5197 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
5198 assert(getWideningDecision(I, VF) == CM_Unknown &&
5199 "Decision should not be set yet.");
5200 auto *Group = getInterleavedAccessGroup(I);
5201 assert(Group && "Must have a group.");
5202
5203 // If the instruction's allocated size doesn't equal it's type size, it
5204 // requires padding and will be scalarized.
5205 auto &DL = I->getModule()->getDataLayout();
5206 auto *ScalarTy = getMemInstValueType(I);
5207 if (hasIrregularType(ScalarTy, DL))
5208 return false;
5209
5210 // Check if masking is required.
5211 // A Group may need masking for one of two reasons: it resides in a block that
5212 // needs predication, or it was decided to use masking to deal with gaps.
5213 bool PredicatedAccessRequiresMasking =
5214 Legal->blockNeedsPredication(I->getParent()) && Legal->isMaskRequired(I);
5215 bool AccessWithGapsRequiresMasking =
5216 Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed();
5217 if (!PredicatedAccessRequiresMasking && !AccessWithGapsRequiresMasking)
5218 return true;
5219
5220 // If masked interleaving is required, we expect that the user/target had
5221 // enabled it, because otherwise it either wouldn't have been created or
5222 // it should have been invalidated by the CostModel.
5223 assert(useMaskedInterleavedAccesses(TTI) &&
5224 "Masked interleave-groups for predicated accesses are not enabled.");
5225
5226 auto *Ty = getMemInstValueType(I);
5227 const Align Alignment = getLoadStoreAlignment(I);
5228 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
5229 : TTI.isLegalMaskedStore(Ty, Alignment);
5230}
5231
5232bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
5233 Instruction *I, ElementCount VF) {
5234 // Get and ensure we have a valid memory instruction.
5235 LoadInst *LI = dyn_cast<LoadInst>(I);
5236 StoreInst *SI = dyn_cast<StoreInst>(I);
5237 assert((LI || SI) && "Invalid memory instruction");
5238
5239 auto *Ptr = getLoadStorePointerOperand(I);
5240
5241 // In order to be widened, the pointer should be consecutive, first of all.
5242 if (!Legal->isConsecutivePtr(Ptr))
5243 return false;
5244
5245 // If the instruction is a store located in a predicated block, it will be
5246 // scalarized.
5247 if (isScalarWithPredication(I))
5248 return false;
5249
5250 // If the instruction's allocated size doesn't equal it's type size, it
5251 // requires padding and will be scalarized.
5252 auto &DL = I->getModule()->getDataLayout();
5253 auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();
5254 if (hasIrregularType(ScalarTy, DL))
5255 return false;
5256
5257 return true;
5258}
5259
5260void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
5261 // We should not collect Uniforms more than once per VF. Right now,
5262 // this function is called from collectUniformsAndScalars(), which
5263 // already does this check. Collecting Uniforms for VF=1 does not make any
5264 // sense.
5265
5266 assert(VF.isVector() && Uniforms.find(VF) == Uniforms.end() &&
5267 "This function should not be visited twice for the same VF");
5268
5269 // Visit the list of Uniforms. If we'll not find any uniform value, we'll
5270 // not analyze again. Uniforms.count(VF) will return 1.
5271 Uniforms[VF].clear();
5272
5273 // We now know that the loop is vectorizable!
5274 // Collect instructions inside the loop that will remain uniform after
5275 // vectorization.
5276
5277 // Global values, params and instructions outside of current loop are out of
5278 // scope.
5279 auto isOutOfScope = [&](Value *V) -> bool {
5280 Instruction *I = dyn_cast<Instruction>(V);
5281 return (!I || !TheLoop->contains(I));
5282 };
5283
5284 SetVector<Instruction *> Worklist;
5285 BasicBlock *Latch = TheLoop->getLoopLatch();
5286
5287 // Instructions that are scalar with predication must not be considered
5288 // uniform after vectorization, because that would create an erroneous
5289 // replicating region where only a single instance out of VF should be formed.
5290 // TODO: optimize such seldom cases if found important, see PR40816.
5291 auto addToWorklistIfAllowed = [&](Instruction *I) -> void {
5292 if (isOutOfScope(I)) {
5293 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
5294 << *I << "\n");
5295 return;
5296 }
5297 if (isScalarWithPredication(I, VF)) {
5298 LLVM_DEBUG(dbgs() << "LV: Found not uniform being ScalarWithPredication: "
5299 << *I << "\n");
5300 return;
5301 }
5302 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
5303 Worklist.insert(I);
5304 };
5305
5306 // Start with the conditional branch. If the branch condition is an
5307 // instruction contained in the loop that is only used by the branch, it is
5308 // uniform.
5309 auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));
5310 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
5311 addToWorklistIfAllowed(Cmp);
5312
5313 auto isUniformDecision = [&](Instruction *I, ElementCount VF) {
5314 InstWidening WideningDecision = getWideningDecision(I, VF);
5315 assert(WideningDecision != CM_Unknown &&
5316 "Widening decision should be ready at this moment");
5317
5318 // A uniform memory op is itself uniform. We exclude uniform stores
5319 // here as they demand the last lane, not the first one.
5320 if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) {
5321 assert(WideningDecision == CM_Scalarize);
5322 return true;
5323 }
5324
5325 return (WideningDecision == CM_Widen ||
5326 WideningDecision == CM_Widen_Reverse ||
5327 WideningDecision == CM_Interleave);
5328 };
5329
5330
5331 // Returns true if Ptr is the pointer operand of a memory access instruction
5332 // I, and I is known to not require scalarization.
5333 auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
5334 return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF);
5335 };
5336
5337 // Holds a list of values which are known to have at least one uniform use.
5338 // Note that there may be other uses which aren't uniform. A "uniform use"
5339 // here is something which only demands lane 0 of the unrolled iterations;
5340 // it does not imply that all lanes produce the same value (e.g. this is not
5341 // the usual meaning of uniform)
5342 SmallPtrSet<Value *, 8> HasUniformUse;
5343
5344 // Scan the loop for instructions which are either a) known to have only
5345 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
5346 for (auto *BB : TheLoop->blocks())
5347 for (auto &I : *BB) {
5348 // If there's no pointer operand, there's nothing to do.
5349 auto *Ptr = getLoadStorePointerOperand(&I);
5350 if (!Ptr)
5351 continue;
5352
5353 // A uniform memory op is itself uniform. We exclude uniform stores
5354 // here as they demand the last lane, not the first one.
5355 if (isa<LoadInst>(I) && Legal->isUniformMemOp(I))
5356 addToWorklistIfAllowed(&I);
5357
5358 if (isUniformDecision(&I, VF)) {
5359 assert(isVectorizedMemAccessUse(&I, Ptr) && "consistency check");
5360 HasUniformUse.insert(Ptr);
5361 }
5362 }
5363
5364 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
5365 // demanding) users. Since loops are assumed to be in LCSSA form, this
5366 // disallows uses outside the loop as well.
5367 for (auto *V : HasUniformUse) {
5368 if (isOutOfScope(V))
5369 continue;
5370 auto *I = cast<Instruction>(V);
5371 auto UsersAreMemAccesses =
5372 llvm::all_of(I->users(), [&](User *U) -> bool {
5373 return isVectorizedMemAccessUse(cast<Instruction>(U), V);
5374 });
5375 if (UsersAreMemAccesses)
5376 addToWorklistIfAllowed(I);
5377 }
5378
5379 // Expand Worklist in topological order: whenever a new instruction
5380 // is added , its users should be already inside Worklist. It ensures
5381 // a uniform instruction will only be used by uniform instructions.
5382 unsigned idx = 0;
5383 while (idx != Worklist.size()) {
5384 Instruction *I = Worklist[idx++];
5385
5386 for (auto OV : I->operand_values()) {
5387 // isOutOfScope operands cannot be uniform instructions.
5388 if (isOutOfScope(OV))
5389 continue;
5390 // First order recurrence Phi's should typically be considered
5391 // non-uniform.
5392 auto *OP = dyn_cast<PHINode>(OV);
5393 if (OP && Legal->isFirstOrderRecurrence(OP))
5394 continue;
5395 // If all the users of the operand are uniform, then add the
5396 // operand into the uniform worklist.
5397 auto *OI = cast<Instruction>(OV);
5398 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
5399 auto *J = cast<Instruction>(U);
5400 return Worklist.count(J) || isVectorizedMemAccessUse(J, OI);
5401 }))
5402 addToWorklistIfAllowed(OI);
5403 }
5404 }
5405
5406 // For an instruction to be added into Worklist above, all its users inside
5407 // the loop should also be in Worklist. However, this condition cannot be
5408 // true for phi nodes that form a cyclic dependence. We must process phi
5409 // nodes separately. An induction variable will remain uniform if all users
5410 // of the induction variable and induction variable update remain uniform.
5411 // The code below handles both pointer and non-pointer induction variables.
5412 for (auto &Induction : Legal->getInductionVars()) {
5413 auto *Ind = Induction.first;
5414 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
5415
5416 // Determine if all users of the induction variable are uniform after
5417 // vectorization.
5418 auto UniformInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
5419 auto *I = cast<Instruction>(U);
5420 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
5421 isVectorizedMemAccessUse(I, Ind);
5422 });
5423 if (!UniformInd)
5424 continue;
5425
5426 // Determine if all users of the induction variable update instruction are
5427 // uniform after vectorization.
5428 auto UniformIndUpdate =
5429 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
5430 auto *I = cast<Instruction>(U);
5431 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
5432 isVectorizedMemAccessUse(I, IndUpdate);
5433 });
5434 if (!UniformIndUpdate)
5435 continue;
5436
5437 // The induction variable and its update instruction will remain uniform.
5438 addToWorklistIfAllowed(Ind);
5439 addToWorklistIfAllowed(IndUpdate);
5440 }
5441
5442 Uniforms[VF].insert(Worklist.begin(), Worklist.end());
5443}
5444
5445bool LoopVectorizationCostModel::runtimeChecksRequired() {
5446 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n");
5447
5448 if (Legal->getRuntimePointerChecking()->Need) {
5449 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
5450 "runtime pointer checks needed. Enable vectorization of this "
5451 "loop with '#pragma clang loop vectorize(enable)' when "
5452 "compiling with -Os/-Oz",
5453 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5454 return true;
5455 }
5456
5457 if (!PSE.getUnionPredicate().getPredicates().empty()) {
5458 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
5459 "runtime SCEV checks needed. Enable vectorization of this "
5460 "loop with '#pragma clang loop vectorize(enable)' when "
5461 "compiling with -Os/-Oz",
5462 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5463 return true;
5464 }
5465
5466 // FIXME: Avoid specializing for stride==1 instead of bailing out.
5467 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
5468 reportVectorizationFailure("Runtime stride check for small trip count",
5469 "runtime stride == 1 checks needed. Enable vectorization of "
5470 "this loop without such check by compiling with -Os/-Oz",
5471 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5472 return true;
5473 }
5474
5475 return false;
5476}
5477
5478Optional<ElementCount>
5479LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
5480 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
5481 // TODO: It may by useful to do since it's still likely to be dynamically
5482 // uniform if the target can skip.
5483 reportVectorizationFailure(
5484 "Not inserting runtime ptr check for divergent target",
5485 "runtime pointer checks needed. Not enabled for divergent target",
5486 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
5487 return None;
5488 }
5489
5490 unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
5491 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
5492 if (TC == 1) {
5493 reportVectorizationFailure("Single iteration (non) loop",
5494 "loop trip count is one, irrelevant for vectorization",
5495 "SingleIterationLoop", ORE, TheLoop);
5496 return None;
5497 }
5498
5499 switch (ScalarEpilogueStatus) {
5500 case CM_ScalarEpilogueAllowed:
5501 return computeFeasibleMaxVF(TC, UserVF);
5502 case CM_ScalarEpilogueNotAllowedUsePredicate:
5503 LLVM_FALLTHROUGH;
5504 case CM_ScalarEpilogueNotNeededUsePredicate:
5505 LLVM_DEBUG(
5506 dbgs() << "LV: vector predicate hint/switch found.\n"
5507 << "LV: Not allowing scalar epilogue, creating predicated "
5508 << "vector loop.\n");
5509 break;
5510 case CM_ScalarEpilogueNotAllowedLowTripLoop:
5511 // fallthrough as a special case of OptForSize
5512 case CM_ScalarEpilogueNotAllowedOptSize:
5513 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
5514 LLVM_DEBUG(
5515 dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
5516 else
5517 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "
5518 << "count.\n");
5519
5520 // Bail if runtime checks are required, which are not good when optimising
5521 // for size.
5522 if (runtimeChecksRequired())
5523 return None;
5524
5525 break;
5526 }
5527
5528 // The only loops we can vectorize without a scalar epilogue, are loops with
5529 // a bottom-test and a single exiting block. We'd have to handle the fact
5530 // that not every instruction executes on the last iteration. This will
5531 // require a lane mask which varies through the vector loop body. (TODO)
5532 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
5533 // If there was a tail-folding hint/switch, but we can't fold the tail by
5534 // masking, fallback to a vectorization with a scalar epilogue.
5535 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
5536 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
5537 "scalar epilogue instead.\n");
5538 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
5539 return computeFeasibleMaxVF(TC, UserVF);
5540 }
5541 return None;
5542 }
5543
5544 // Now try the tail folding
5545
5546 // Invalidate interleave groups that require an epilogue if we can't mask
5547 // the interleave-group.
5548 if (!useMaskedInterleavedAccesses(TTI)) {
5549 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
5550 "No decisions should have been taken at this point");
5551 // Note: There is no need to invalidate any cost modeling decisions here, as
5552 // non where taken so far.
5553 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
5554 }
5555
5556 ElementCount MaxVF = computeFeasibleMaxVF(TC, UserVF);
5557 assert(!MaxVF.isScalable() &&
5558 "Scalable vectors do not yet support tail folding");
5559 assert((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) &&
5560 "MaxVF must be a power of 2");
5561 unsigned MaxVFtimesIC =
5562 UserIC ? MaxVF.getFixedValue() * UserIC : MaxVF.getFixedValue();
5563 // Avoid tail folding if the trip count is known to be a multiple of any VF we
5564 // chose.
5565 ScalarEvolution *SE = PSE.getSE();
5566 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
5567 const SCEV *ExitCount = SE->getAddExpr(
5568 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
5569 const SCEV *Rem = SE->getURemExpr(
5570 ExitCount, SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
5571 if (Rem->isZero()) {
5572 // Accept MaxVF if we do not have a tail.
5573 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
5574 return MaxVF;
5575 }
5576
5577 // If we don't know the precise trip count, or if the trip count that we
5578 // found modulo the vectorization factor is not zero, try to fold the tail
5579 // by masking.
5580 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
5581 if (Legal->prepareToFoldTailByMasking()) {
5582 FoldTailByMasking = true;
5583 return MaxVF;
5584 }
5585
5586 // If there was a tail-folding hint/switch, but we can't fold the tail by
5587 // masking, fallback to a vectorization with a scalar epilogue.
5588 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
5589 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
5590 "scalar epilogue instead.\n");
5591 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
5592 return MaxVF;
5593 }
5594
5595 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
5596 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
5597 return None;
5598 }
5599
5600 if (TC == 0) {
5601 reportVectorizationFailure(
5602 "Unable to calculate the loop count due to complex control flow",
5603 "unable to calculate the loop count due to complex control flow",
5604 "UnknownLoopCountComplexCFG", ORE, TheLoop);
5605 return None;
5606 }
5607
5608 reportVectorizationFailure(
5609 "Cannot optimize for size and vectorize at the same time.",
5610 "cannot optimize for size and vectorize at the same time. "
5611 "Enable vectorization of this loop with '#pragma clang loop "
5612 "vectorize(enable)' when compiling with -Os/-Oz",
5613 "NoTailLoopWithOptForSize", ORE, TheLoop);
5614 return None;
5615}
5616
5617ElementCount
5618LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
5619 ElementCount UserVF) {
5620 bool IgnoreScalableUserVF = UserVF.isScalable() &&
5621 !TTI.supportsScalableVectors() &&
5622 !ForceTargetSupportsScalableVectors;
5623 if (IgnoreScalableUserVF) {
5624 LLVM_DEBUG(
5625 dbgs() << "LV: Ignoring VF=" << UserVF
5626 << " because target does not support scalable vectors.\n");
5627 ORE->emit([&]() {
5628 return OptimizationRemarkAnalysis(DEBUG_TYPE, "IgnoreScalableUserVF",
5629 TheLoop->getStartLoc(),
5630 TheLoop->getHeader())
5631 << "Ignoring VF=" << ore::NV("UserVF", UserVF)
5632 << " because target does not support scalable vectors.";
5633 });
5634 }
5635
5636 // Beyond this point two scenarios are handled. If UserVF isn't specified
5637 // then a suitable VF is chosen. If UserVF is specified and there are
5638 // dependencies, check if it's legal. However, if a UserVF is specified and
5639 // there are no dependencies, then there's nothing to do.
5640 if (UserVF.isNonZero() && !IgnoreScalableUserVF &&
5641 Legal->isSafeForAnyVectorWidth())
5642 return UserVF;
5643
5644 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
5645 unsigned SmallestType, WidestType;
5646 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
5647 unsigned WidestRegister = TTI.getRegisterBitWidth(true);
5648
5649 // Get the maximum safe dependence distance in bits computed by LAA.
5650 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
5651 // the memory accesses that is most restrictive (involved in the smallest
5652 // dependence distance).
5653 unsigned MaxSafeVectorWidthInBits = Legal->getMaxSafeVectorWidthInBits();
5654
5655 // If the user vectorization factor is legally unsafe, clamp it to a safe
5656 // value. Otherwise, return as is.
5657 if (UserVF.isNonZero() && !IgnoreScalableUserVF) {
5658 unsigned MaxSafeElements =
5659 PowerOf2Floor(MaxSafeVectorWidthInBits / WidestType);
5660 ElementCount MaxSafeVF = ElementCount::getFixed(MaxSafeElements);
5661
5662 if (UserVF.isScalable()) {
5663 Optional<unsigned> MaxVScale = TTI.getMaxVScale();
5664
5665 // Scale VF by vscale before checking if it's safe.
5666 MaxSafeVF = ElementCount::getScalable(
5667 MaxVScale ? (MaxSafeElements / MaxVScale.getValue()) : 0);
5668
5669 if (MaxSafeVF.isZero()) {
5670 // The dependence distance is too small to use scalable vectors,
5671 // fallback on fixed.
5672 LLVM_DEBUG(
5673 dbgs()
5674 << "LV: Max legal vector width too small, scalable vectorization "
5675 "unfeasible. Using fixed-width vectorization instead.\n");
5676 ORE->emit([&]() {
5677 return OptimizationRemarkAnalysis(DEBUG_TYPE, "ScalableVFUnfeasible",
5678 TheLoop->getStartLoc(),
5679 TheLoop->getHeader())
5680 << "Max legal vector width too small, scalable vectorization "
5681 << "unfeasible. Using fixed-width vectorization instead.";
5682 });
5683 return computeFeasibleMaxVF(
5684 ConstTripCount, ElementCount::getFixed(UserVF.getKnownMinValue()));
5685 }
5686 }
5687
5688 LLVM_DEBUG(dbgs() << "LV: The max safe VF is: " << MaxSafeVF << ".\n");
5689
5690 if (ElementCount::isKnownLE(UserVF, MaxSafeVF))
5691 return UserVF;
5692
5693 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
5694 << " is unsafe, clamping to max safe VF=" << MaxSafeVF
5695 << ".\n");
5696 ORE->emit([&]() {
5697 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
5698 TheLoop->getStartLoc(),
5699 TheLoop->getHeader())
5700 << "User-specified vectorization factor "
5701 << ore::NV("UserVectorizationFactor", UserVF)
5702 << " is unsafe, clamping to maximum safe vectorization factor "
5703 << ore::NV("VectorizationFactor", MaxSafeVF);
5704 });
5705 return MaxSafeVF;
5706 }
5707
5708 WidestRegister = std::min(WidestRegister, MaxSafeVectorWidthInBits);
5709
5710 // Ensure MaxVF is a power of 2; the dependence distance bound may not be.
5711 // Note that both WidestRegister and WidestType may not be a powers of 2.
5712 unsigned MaxVectorSize = PowerOf2Floor(WidestRegister / WidestType);
5713
5714 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType
5715 << " / " << WidestType << " bits.\n");
5716 LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "
5717 << WidestRegister << " bits.\n");
5718
5719 assert(MaxVectorSize <= WidestRegister &&
5720 "Did not expect to pack so many elements"
5721 " into one vector!");
5722 if (MaxVectorSize == 0) {
5723 LLVM_DEBUG(dbgs() << "LV: The target has no vector registers.\n");
5724 MaxVectorSize = 1;
5725 return ElementCount::getFixed(MaxVectorSize);
5726 } else if (ConstTripCount && ConstTripCount < MaxVectorSize &&
5727 isPowerOf2_32(ConstTripCount)) {
5728 // We need to clamp the VF to be the ConstTripCount. There is no point in
5729 // choosing a higher viable VF as done in the loop below.
5730 LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to the constant trip count: "
5731 << ConstTripCount << "\n");
5732 MaxVectorSize = ConstTripCount;
5733 return ElementCount::getFixed(MaxVectorSize);
5734 }
5735
5736 unsigned MaxVF = MaxVectorSize;
5737 if (TTI.shouldMaximizeVectorBandwidth(!isScalarEpilogueAllowed()) ||
5738 (MaximizeBandwidth && isScalarEpilogueAllowed())) {
5739 // Collect all viable vectorization factors larger than the default MaxVF
5740 // (i.e. MaxVectorSize).
5741 SmallVector<ElementCount, 8> VFs;
5742 unsigned NewMaxVectorSize = WidestRegister / SmallestType;
5743 for (unsigned VS = MaxVectorSize * 2; VS <= NewMaxVectorSize; VS *= 2)
5744 VFs.push_back(ElementCount::getFixed(VS));
5745
5746 // For each VF calculate its register usage.
5747 auto RUs = calculateRegisterUsage(VFs);
5748
5749 // Select the largest VF which doesn't require more registers than existing
5750 // ones.
5751 for (int i = RUs.size() - 1; i >= 0; --i) {
5752 bool Selected = true;
5753 for (auto& pair : RUs[i].MaxLocalUsers) {
5754 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first);
5755 if (pair.second > TargetNumRegisters)
5756 Selected = false;
5757 }
5758 if (Selected) {
5759 MaxVF = VFs[i].getKnownMinValue();
5760 break;
5761 }
5762 }
5763 if (unsigned MinVF = TTI.getMinimumVF(SmallestType)) {
5764 if (MaxVF < MinVF) {
5765 LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVF
5766 << ") with target's minimum: " << MinVF << '\n');
5767 MaxVF = MinVF;
5768 }
5769 }
5770 }
5771 return ElementCount::getFixed(MaxVF);
5772}
5773
5774VectorizationFactor
5775LoopVectorizationCostModel::selectVectorizationFactor(ElementCount MaxVF) {
5776 // FIXME: This can be fixed for scalable vectors later, because at this stage
5777 // the LoopVectorizer will only consider vectorizing a loop with scalable
5778 // vectors when the loop has a hint to enable vectorization for a given VF.
5779 assert(!MaxVF.isScalable() && "scalable vectors not yet supported");
5780
5781 InstructionCost ExpectedCost = expectedCost(ElementCount::getFixed(1)).first;
5782 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n");
5783 assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop");
5784
5785 unsigned Width = 1;
5786 const float ScalarCost = *ExpectedCost.getValue();
5787 float Cost = ScalarCost;
5788
5789 bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
5790 if (ForceVectorization && MaxVF.isVector()) {
5791 // Ignore scalar width, because the user explicitly wants vectorization.
5792 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
5793 // evaluation.
5794 Cost = std::numeric_limits<float>::max();
5795 }
5796
5797 for (unsigned i = 2; i <= MaxVF.getFixedValue(); i *= 2) {
5798 // Notice that the vector loop needs to be executed less times, so
5799 // we need to divide the cost of the vector loops by the width of
5800 // the vector elements.
5801 VectorizationCostTy C = expectedCost(ElementCount::getFixed(i));
5802 assert(C.first.isValid() && "Unexpected invalid cost for vector loop");
5803 float VectorCost = *C.first.getValue() / (float)i;
5804 LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << i
5805 << " costs: " << (int)VectorCost << ".\n");
5806 if (!C.second && !ForceVectorization) {
5807 LLVM_DEBUG(
5808 dbgs() << "LV: Not considering vector loop of width " << i
5809 << " because it will not generate any vector instructions.\n");
5810 continue;
5811 }
5812
5813 // If profitable add it to ProfitableVF list.
5814 if (VectorCost < ScalarCost) {
5815 ProfitableVFs.push_back(VectorizationFactor(
5816 {ElementCount::getFixed(i), (unsigned)VectorCost}));
5817 }
5818
5819 if (VectorCost < Cost) {
5820 Cost = VectorCost;
5821 Width = i;
5822 }
5823 }
5824
5825 if (!EnableCondStoresVectorization && NumPredStores) {
5826 reportVectorizationFailure("There are conditional stores.",
5827 "store that is conditionally executed prevents vectorization",
5828 "ConditionalStore", ORE, TheLoop);
5829 Width = 1;
5830 Cost = ScalarCost;
5831 }
5832
5833 LLVM_DEBUG(if (ForceVectorization && Width > 1 && Cost >= ScalarCost) dbgs()
5834 << "LV: Vectorization seems to be not beneficial, "
5835 << "but was forced by a user.\n");
5836 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << Width << ".\n");
5837 VectorizationFactor Factor = {ElementCount::getFixed(Width),
5838 (unsigned)(Width * Cost)};
5839 return Factor;
5840}
5841
5842bool LoopVectorizationCostModel::isCandidateForEpilogueVectorization(
5843 const Loop &L, ElementCount VF) const {
5844 // Cross iteration phis such as reductions need special handling and are
5845 // currently unsupported.
5846 if (any_of(L.getHeader()->phis(), [&](PHINode &Phi) {
5847 return Legal->isFirstOrderRecurrence(&Phi) ||
5848 Legal->isReductionVariable(&Phi);
5849 }))
5850 return false;
5851
5852 // Phis with uses outside of the loop require special handling and are
5853 // currently unsupported.
5854 for (auto &Entry : Legal->getInductionVars()) {
5855 // Look for uses of the value of the induction at the last iteration.
5856 Value *PostInc = Entry.first->getIncomingValueForBlock(L.getLoopLatch());
5857 for (User *U : PostInc->users())
5858 if (!L.contains(cast<Instruction>(U)))
5859 return false;
5860 // Look for uses of penultimate value of the induction.
5861 for (User *U : Entry.first->users())
5862 if (!L.contains(cast<Instruction>(U)))
5863 return false;
5864 }
5865
5866 // Induction variables that are widened require special handling that is
5867 // currently not supported.
5868 if (any_of(Legal->getInductionVars(), [&](auto &Entry) {
5869 return !(this->isScalarAfterVectorization(Entry.first, VF) ||
5870 this->isProfitableToScalarize(Entry.first, VF));
5871 }))
5872 return false;
5873
5874 return true;
5875}
5876
5877bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
5878 const ElementCount VF) const {
5879 // FIXME: We need a much better cost-model to take different parameters such
5880 // as register pressure, code size increase and cost of extra branches into
5881 // account. For now we apply a very crude heuristic and only consider loops
5882 // with vectorization factors larger than a certain value.
5883 // We also consider epilogue vectorization unprofitable for targets that don't
5884 // consider interleaving beneficial (eg. MVE).
5885 if (TTI.getMaxInterleaveFactor(VF.getKnownMinValue()) <= 1)
5886 return false;
5887 if (VF.getFixedValue() >= EpilogueVectorizationMinVF)
5888 return true;
5889 return false;
5890}
5891
5892VectorizationFactor
5893LoopVectorizationCostModel::selectEpilogueVectorizationFactor(
5894 const ElementCount MainLoopVF, const LoopVectorizationPlanner &LVP) {
5895 VectorizationFactor Result = VectorizationFactor::Disabled();
5896 if (!EnableEpilogueVectorization) {
5897 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n";);
5898 return Result;
5899 }
5900
5901 if (!isScalarEpilogueAllowed()) {
5902 LLVM_DEBUG(
5903 dbgs() << "LEV: Unable to vectorize epilogue because no epilogue is "
5904 "allowed.\n";);
5905 return Result;
5906 }
5907
5908 // FIXME: This can be fixed for scalable vectors later, because at this stage
5909 // the LoopVectorizer will only consider vectorizing a loop with scalable
5910 // vectors when the loop has a hint to enable vectorization for a given VF.
5911 if (MainLoopVF.isScalable()) {
5912 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization for scalable vectors not "
5913 "yet supported.\n");
5914 return Result;
5915 }
5916
5917 // Not really a cost consideration, but check for unsupported cases here to
5918 // simplify the logic.
5919 if (!isCandidateForEpilogueVectorization(*TheLoop, MainLoopVF)) {
5920 LLVM_DEBUG(
5921 dbgs() << "LEV: Unable to vectorize epilogue because the loop is "
5922 "not a supported candidate.\n";);
5923 return Result;
5924 }
5925
5926 if (EpilogueVectorizationForceVF > 1) {
5927 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n";);
5928 if (LVP.hasPlanWithVFs(
5929 {MainLoopVF, ElementCount::getFixed(EpilogueVectorizationForceVF)}))
5930 return {ElementCount::getFixed(EpilogueVectorizationForceVF), 0};
5931 else {
5932 LLVM_DEBUG(
5933 dbgs()
5934 << "LEV: Epilogue vectorization forced factor is not viable.\n";);
5935 return Result;
5936 }
5937 }
5938
5939 if (TheLoop->getHeader()->getParent()->hasOptSize() ||
5940 TheLoop->getHeader()->getParent()->hasMinSize()) {
5941 LLVM_DEBUG(
5942 dbgs()
5943 << "LEV: Epilogue vectorization skipped due to opt for size.\n";);
5944 return Result;
5945 }
5946
5947 if (!isEpilogueVectorizationProfitable(MainLoopVF))
5948 return Result;
5949
5950 for (auto &NextVF : ProfitableVFs)
5951 if (ElementCount::isKnownLT(NextVF.Width, MainLoopVF) &&
5952 (Result.Width.getFixedValue() == 1 || NextVF.Cost < Result.Cost) &&
5953 LVP.hasPlanWithVFs({MainLoopVF, NextVF.Width}))
5954 Result = NextVF;
5955
5956 if (Result != VectorizationFactor::Disabled())
5957 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
5958 << Result.Width.getFixedValue() << "\n";);
5959 return Result;
5960}
5961
5962std::pair<unsigned, unsigned>
5963LoopVectorizationCostModel::getSmallestAndWidestTypes() {
5964 unsigned MinWidth = -1U;
5965 unsigned MaxWidth = 8;
5966 const DataLayout &DL = TheFunction->getParent()->getDataLayout();
5967
5968 // For each block.
5969 for (BasicBlock *BB : TheLoop->blocks()) {
5970 // For each instruction in the loop.
5971 for (Instruction &I : BB->instructionsWithoutDebug()) {
5972 Type *T = I.getType();
5973
5974 // Skip ignored values.
5975 if (ValuesToIgnore.count(&I))
5976 continue;
5977
5978 // Only examine Loads, Stores and PHINodes.
5979 if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I))
5980 continue;
5981
5982 // Examine PHI nodes that are reduction variables. Update the type to
5983 // account for the recurrence type.
5984 if (auto *PN = dyn_cast<PHINode>(&I)) {
5985 if (!Legal->isReductionVariable(PN))
5986 continue;
5987 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[PN];
5988 if (PreferInLoopReductions ||
5989 TTI.preferInLoopReduction(RdxDesc.getOpcode(),
5990 RdxDesc.getRecurrenceType(),
5991 TargetTransformInfo::ReductionFlags()))
5992 continue;
5993 T = RdxDesc.getRecurrenceType();
5994 }
5995
5996 // Examine the stored values.
5997 if (auto *ST = dyn_cast<StoreInst>(&I))
5998 T = ST->getValueOperand()->getType();
5999
6000 // Ignore loaded pointer types and stored pointer types that are not
6001 // vectorizable.
6002 //
6003 // FIXME: The check here attempts to predict whether a load or store will
6004 // be vectorized. We only know this for certain after a VF has
6005 // been selected. Here, we assume that if an access can be
6006 // vectorized, it will be. We should also look at extending this
6007 // optimization to non-pointer types.
6008 //
6009 if (T->isPointerTy() && !isConsecutiveLoadOrStore(&I) &&
6010 !isAccessInterleaved(&I) && !isLegalGatherOrScatter(&I))
6011 continue;
6012
6013 MinWidth = std::min(MinWidth,
6014 (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
6015 MaxWidth = std::max(MaxWidth,
6016 (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
6017 }
6018 }
6019
6020 return {MinWidth, MaxWidth};
6021}
6022
6023unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
6024 unsigned LoopCost) {
6025 // -- The interleave heuristics --
6026 // We interleave the loop in order to expose ILP and reduce the loop overhead.
6027 // There are many micro-architectural considerations that we can't predict
6028 // at this level. For example, frontend pressure (on decode or fetch) due to
6029 // code size, or the number and capabilities of the execution ports.
6030 //
6031 // We use the following heuristics to select the interleave count:
6032 // 1. If the code has reductions, then we interleave to break the cross
6033 // iteration dependency.
6034 // 2. If the loop is really small, then we interleave to reduce the loop
6035 // overhead.
6036 // 3. We don't interleave if we think that we will spill registers to memory
6037 // due to the increased register pressure.
6038
6039 if (!isScalarEpilogueAllowed())
6040 return 1;
6041
6042 // We used the distance for the interleave count.
6043 if (Legal->getMaxSafeDepDistBytes() != -1U)
6044 return 1;
6045
6046 auto BestKnownTC = getSmallBestKnownTC(*PSE.getSE(), TheLoop);
6047 const bool HasReductions = !Legal->getReductionVars().empty();
6048 // Do not interleave loops with a relatively small known or estimated trip
6049 // count. But we will interleave when InterleaveSmallLoopScalarReduction is
6050 // enabled, and the code has scalar reductions(HasReductions && VF = 1),
6051 // because with the above conditions interleaving can expose ILP and break
6052 // cross iteration dependences for reductions.
6053 if (BestKnownTC && (*BestKnownTC < TinyTripCountInterleaveThreshold) &&
6054 !(InterleaveSmallLoopScalarReduction && HasReductions && VF.isScalar()))
6055 return 1;
6056
6057 RegisterUsage R = calculateRegisterUsage({VF})[0];
6058 // We divide by these constants so assume that we have at least one
6059 // instruction that uses at least one register.
6060 for (auto& pair : R.MaxLocalUsers) {
6061 pair.second = std::max(pair.second, 1U);
6062 }
6063
6064 // We calculate the interleave count using the following formula.
6065 // Subtract the number of loop invariants from the number of available
6066 // registers. These registers are used by all of the interleaved instances.
6067 // Next, divide the remaining registers by the number of registers that is
6068 // required by the loop, in order to estimate how many parallel instances
6069 // fit without causing spills. All of this is rounded down if necessary to be
6070 // a power of two. We want power of two interleave count to simplify any
6071 // addressing operations or alignment considerations.
6072 // We also want power of two interleave counts to ensure that the induction
6073 // variable of the vector loop wraps to zero, when tail is folded by masking;
6074 // this currently happens when OptForSize, in which case IC is set to 1 above.
6075 unsigned IC = UINT_MAX;
6076
6077 for (auto& pair : R.MaxLocalUsers) {
6078 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first);
6079 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
6080 << " registers of "
6081 << TTI.getRegisterClassName(pair.first) << " register class\n");
6082 if (VF.isScalar()) {
6083 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
6084 TargetNumRegisters = ForceTargetNumScalarRegs;
6085 } else {
6086 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
6087 TargetNumRegisters = ForceTargetNumVectorRegs;
6088 }
6089 unsigned MaxLocalUsers = pair.second;
6090 unsigned LoopInvariantRegs = 0;
6091 if (R.LoopInvariantRegs.find(pair.first) != R.LoopInvariantRegs.end())
6092 LoopInvariantRegs = R.LoopInvariantRegs[pair.first];
6093
6094 unsigned TmpIC = PowerOf2Floor((TargetNumRegisters - LoopInvariantRegs) / MaxLocalUsers);
6095 // Don't count the induction variable as interleaved.
6096 if (EnableIndVarRegisterHeur) {
6097 TmpIC =
6098 PowerOf2Floor((TargetNumRegisters - LoopInvariantRegs - 1) /
6099 std::max(1U, (MaxLocalUsers - 1)));
6100 }
6101
6102 IC = std::min(IC, TmpIC);
6103 }
6104
6105 // Clamp the interleave ranges to reasonable counts.
6106 unsigned MaxInterleaveCount =
6107 TTI.getMaxInterleaveFactor(VF.getKnownMinValue());
6108
6109 // Check if the user has overridden the max.
6110 if (VF.isScalar()) {
6111 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
6112 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
6113 } else {
6114 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
6115 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
6116 }
6117
6118 // If trip count is known or estimated compile time constant, limit the
6119 // interleave count to be less than the trip count divided by VF, provided it
6120 // is at least 1.
6121 //
6122 // For scalable vectors we can't know if interleaving is beneficial. It may
6123 // not be beneficial for small loops if none of the lanes in the second vector
6124 // iterations is enabled. However, for larger loops, there is likely to be a
6125 // similar benefit as for fixed-width vectors. For now, we choose to leave
6126 // the InterleaveCount as if vscale is '1', although if some information about
6127 // the vector is known (e.g. min vector size), we can make a better decision.
6128 if (BestKnownTC) {
6129 MaxInterleaveCount =
6130 std::min(*BestKnownTC / VF.getKnownMinValue(), MaxInterleaveCount);
6131 // Make sure MaxInterleaveCount is greater than 0.
6132 MaxInterleaveCount = std::max(1u, MaxInterleaveCount);
6133 }
6134
6135 assert(MaxInterleaveCount > 0 &&
6136 "Maximum interleave count must be greater than 0");
6137
6138 // Clamp the calculated IC to be between the 1 and the max interleave count
6139 // that the target and trip count allows.
6140 if (IC > MaxInterleaveCount)
6141 IC = MaxInterleaveCount;
6142 else
6143 // Make sure IC is greater than 0.
6144 IC = std::max(1u, IC);
6145
6146 assert(IC > 0 && "Interleave count must be greater than 0.");
6147
6148 // If we did not calculate the cost for VF (because the user selected the VF)
6149 // then we calculate the cost of VF here.
6150 if (LoopCost == 0) {
6151 assert(expectedCost(VF).first.isValid() && "Expected a valid cost");
6152 LoopCost = *expectedCost(VF).first.getValue();
6153 }
6154
6155 assert(LoopCost && "Non-zero loop cost expected");
6156
6157 // Interleave if we vectorized this loop and there is a reduction that could
6158 // benefit from interleaving.
6159 if (VF.isVector() && HasReductions) {
6160 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
6161 return IC;
6162 }
6163
6164 // Note that if we've already vectorized the loop we will have done the
6165 // runtime check and so interleaving won't require further checks.
6166 bool InterleavingRequiresRuntimePointerCheck =
6167 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
6168
6169 // We want to interleave small loops in order to reduce the loop overhead and
6170 // potentially expose ILP opportunities.
6171 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
6172 << "LV: IC is " << IC << '\n'
6173 << "LV: VF is " << VF << '\n');
6174 const bool AggressivelyInterleaveReductions =
6175 TTI.enableAggressiveInterleaving(HasReductions);
6176 if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {
6177 // We assume that the cost overhead is 1 and we use the cost model
6178 // to estimate the cost of the loop and interleave until the cost of the
6179 // loop overhead is about 5% of the cost of the loop.
6180 unsigned SmallIC =
6181 std::min(IC, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost));
6182
6183 // Interleave until store/load ports (estimated by max interleave count) are
6184 // saturated.
6185 unsigned NumStores = Legal->getNumStores();
6186 unsigned NumLoads = Legal->getNumLoads();
6187 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
6188 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
6189
6190 // If we have a scalar reduction (vector reductions are already dealt with
6191 // by this point), we can increase the critical path length if the loop
6192 // we're interleaving is inside another loop. Limit, by default to 2, so the
6193 // critical path only gets increased by one reduction operation.
6194 if (HasReductions && TheLoop->getLoopDepth() > 1) {
6195 unsigned F = static_cast<unsigned>(MaxNestedScalarReductionIC);
6196 SmallIC = std::min(SmallIC, F);
6197 StoresIC = std::min(StoresIC, F);
6198 LoadsIC = std::min(LoadsIC, F);
6199 }
6200
6201 if (EnableLoadStoreRuntimeInterleave &&
6202 std::max(StoresIC, LoadsIC) > SmallIC) {
6203 LLVM_DEBUG(
6204 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
6205 return std::max(StoresIC, LoadsIC);
6206 }
6207
6208 // If there are scalar reductions and TTI has enabled aggressive
6209 // interleaving for reductions, we will interleave to expose ILP.
6210 if (InterleaveSmallLoopScalarReduction && VF.isScalar() &&
6211 AggressivelyInterleaveReductions) {
6212 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
6213 // Interleave no less than SmallIC but not as aggressive as the normal IC
6214 // to satisfy the rare situation when resources are too limited.
6215 return std::max(IC / 2, SmallIC);
6216 } else {
6217 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
6218 return SmallIC;
6219 }
6220 }
6221
6222 // Interleave if this is a large loop (small loops are already dealt with by
6223 // this point) that could benefit from interleaving.
6224 if (AggressivelyInterleaveReductions) {
6225 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
6226 return IC;
6227 }
6228
6229 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
6230 return 1;
6231}
6232
6233SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>
6234LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<ElementCount> VFs) {
6235 // This function calculates the register usage by measuring the highest number
6236 // of values that are alive at a single location. Obviously, this is a very
6237 // rough estimation. We scan the loop in a topological order in order and
6238 // assign a number to each instruction. We use RPO to ensure that defs are
6239 // met before their users. We assume that each instruction that has in-loop
6240 // users starts an interval. We record every time that an in-loop value is
6241 // used, so we have a list of the first and last occurrences of each
6242 // instruction. Next, we transpose this data structure into a multi map that
6243 // holds the list of intervals that *end* at a specific location. This multi
6244 // map allows us to perform a linear search. We scan the instructions linearly
6245 // and record each time that a new interval starts, by placing it in a set.
6246 // If we find this value in the multi-map then we remove it from the set.
6247 // The max register usage is the maximum size of the set.
6248 // We also search for instructions that are defined outside the loop, but are
6249 // used inside the loop. We need this number separately from the max-interval
6250 // usage number because when we unroll, loop-invariant values do not take
6251 // more register.
6252 LoopBlocksDFS DFS(TheLoop);
6253 DFS.perform(LI);
6254
6255 RegisterUsage RU;
6256
6257 // Each 'key' in the map opens a new interval. The values
6258 // of the map are the index of the 'last seen' usage of the
6259 // instruction that is the key.
6260 using IntervalMap = DenseMap<Instruction *, unsigned>;
6261
6262 // Maps instruction to its index.
6263 SmallVector<Instruction *, 64> IdxToInstr;
6264 // Marks the end of each interval.
6265 IntervalMap EndPoint;
6266 // Saves the list of instruction indices that are used in the loop.
6267 SmallPtrSet<Instruction *, 8> Ends;
6268 // Saves the list of values that are used in the loop but are
6269 // defined outside the loop, such as arguments and constants.
6270 SmallPtrSet<Value *, 8> LoopInvariants;
6271
6272 for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
6273 for (Instruction &I : BB->instructionsWithoutDebug()) {
6274 IdxToInstr.push_back(&I);
6275
6276 // Save the end location of each USE.
6277 for (Value *U : I.operands()) {
6278 auto *Instr = dyn_cast<Instruction>(U);
6279
6280 // Ignore non-instruction values such as arguments, constants, etc.
6281 if (!Instr)
6282 continue;
6283
6284 // If this instruction is outside the loop then record it and continue.
6285 if (!TheLoop->contains(Instr)) {
6286 LoopInvariants.insert(Instr);
6287 continue;
6288 }
6289
6290 // Overwrite previous end points.
6291 EndPoint[Instr] = IdxToInstr.size();
6292 Ends.insert(Instr);
6293 }
6294 }
6295 }
6296
6297 // Saves the list of intervals that end with the index in 'key'.
6298 using InstrList = SmallVector<Instruction *, 2>;
6299 DenseMap<unsigned, InstrList> TransposeEnds;
6300
6301 // Transpose the EndPoints to a list of values that end at each index.
6302 for (auto &Interval : EndPoint)
6303 TransposeEnds[Interval.second].push_back(Interval.first);
6304
6305 SmallPtrSet<Instruction *, 8> OpenIntervals;
6306 SmallVector<RegisterUsage, 8> RUs(VFs.size());
6307 SmallVector<SmallMapVector<unsigned, unsigned, 4>, 8> MaxUsages(VFs.size());
6308
6309 LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n");
6310
6311 // A lambda that gets the register usage for the given type and VF.
6312 const auto &TTICapture = TTI;
6313 auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) {
6314 if (Ty->isTokenTy() || !VectorType::isValidElementType(Ty))
6315 return 0U;
6316 return TTICapture.getRegUsageForType(VectorType::get(Ty, VF));
6317 };
6318
6319 for (unsigned int i = 0, s = IdxToInstr.size(); i < s; ++i) {
6320 Instruction *I = IdxToInstr[i];
6321
6322 // Remove all of the instructions that end at this location.
6323 InstrList &List = TransposeEnds[i];
6324 for (Instruction *ToRemove : List)
6325 OpenIntervals.erase(ToRemove);
6326
6327 // Ignore instructions that are never used within the loop.
6328 if (!Ends.count(I))
6329 continue;
6330
6331 // Skip ignored values.
6332 if (ValuesToIgnore.count(I))
6333 continue;
6334
6335 // For each VF find the maximum usage of registers.
6336 for (unsigned j = 0, e = VFs.size(); j < e; ++j) {
6337 // Count the number of live intervals.
6338 SmallMapVector<unsigned, unsigned, 4> RegUsage;
6339
6340 if (VFs[j].isScalar()) {
6341 for (auto Inst : OpenIntervals) {
6342 unsigned ClassID = TTI.getRegisterClassForType(false, Inst->getType());
6343 if (RegUsage.find(ClassID) == RegUsage.end())
6344 RegUsage[ClassID] = 1;
6345 else
6346 RegUsage[ClassID] += 1;
6347 }
6348 } else {
6349 collectUniformsAndScalars(VFs[j]);
6350 for (auto Inst : OpenIntervals) {
6351 // Skip ignored values for VF > 1.
6352 if (VecValuesToIgnore.count(Inst))
6353 continue;
6354 if (isScalarAfterVectorization(Inst, VFs[j])) {
6355 unsigned ClassID = TTI.getRegisterClassForType(false, Inst->getType());
6356 if (RegUsage.find(ClassID) == RegUsage.end())
6357 RegUsage[ClassID] = 1;
6358 else
6359 RegUsage[ClassID] += 1;
6360 } else {
6361 unsigned ClassID = TTI.getRegisterClassForType(true, Inst->getType());
6362 if (RegUsage.find(ClassID) == RegUsage.end())
6363 RegUsage[ClassID] = GetRegUsage(Inst->getType(), VFs[j]);
6364 else
6365 RegUsage[ClassID] += GetRegUsage(Inst->getType(), VFs[j]);
6366 }
6367 }
6368 }
6369
6370 for (auto& pair : RegUsage) {
6371 if (MaxUsages[j].find(pair.first) != MaxUsages[j].end())
6372 MaxUsages[j][pair.first] = std::max(MaxUsages[j][pair.first], pair.second);
6373 else
6374 MaxUsages[j][pair.first] = pair.second;
6375 }
6376 }
6377
6378 LLVM_DEBUG(dbgs() << "LV(REG): At #" << i << " Interval # "
6379 << OpenIntervals.size() << '\n');
6380
6381 // Add the current instruction to the list of open intervals.
6382 OpenIntervals.insert(I);
6383 }
6384
6385 for (unsigned i = 0, e = VFs.size(); i < e; ++i) {
6386 SmallMapVector<unsigned, unsigned, 4> Invariant;
6387
6388 for (auto Inst : LoopInvariants) {
6389 unsigned Usage =
6390 VFs[i].isScalar() ? 1 : GetRegUsage(Inst->getType(), VFs[i]);
6391 unsigned ClassID =
6392 TTI.getRegisterClassForType(VFs[i].isVector(), Inst->getType());
6393 if (Invariant.find(ClassID) == Invariant.end())
6394 Invariant[ClassID] = Usage;
6395 else
6396 Invariant[ClassID] += Usage;
6397 }
6398
6399 LLVM_DEBUG({
6400 dbgs() << "LV(REG): VF = " << VFs[i] << '\n';
6401 dbgs() << "LV(REG): Found max usage: " << MaxUsages[i].size()
6402 << " item\n";
6403 for (const auto &pair : MaxUsages[i]) {
6404 dbgs() << "LV(REG): RegisterClass: "
6405 << TTI.getRegisterClassName(pair.first) << ", " << pair.second
6406 << " registers\n";
6407 }
6408 dbgs() << "LV(REG): Found invariant usage: " << Invariant.size()
6409 << " item\n";
6410 for (const auto &pair : Invariant) {
6411 dbgs() << "LV(REG): RegisterClass: "
6412 << TTI.getRegisterClassName(pair.first) << ", " << pair.second
6413 << " registers\n";
6414 }
6415 });
6416
6417 RU.LoopInvariantRegs = Invariant;
6418 RU.MaxLocalUsers = MaxUsages[i];
6419 RUs[i] = RU;
6420 }
6421
6422 return RUs;
6423}
6424
6425bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I){
6426 // TODO: Cost model for emulated masked load/store is completely
6427 // broken. This hack guides the cost model to use an artificially
6428 // high enough value to practically disable vectorization with such
6429 // operations, except where previously deployed legality hack allowed
6430 // using very low cost values. This is to avoid regressions coming simply
6431 // from moving "masked load/store" check from legality to cost model.
6432 // Masked Load/Gather emulation was previously never allowed.
6433 // Limited number of Masked Store/Scatter emulation was allowed.
6434 assert(isPredicatedInst(I) && "Expecting a scalar emulated instruction");
6435 return isa<LoadInst>(I) ||
6436 (isa<StoreInst>(I) &&
6437 NumPredStores > NumberOfStoresToPredicate);
6438}
6439
6440void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
6441 // If we aren't vectorizing the loop, or if we've already collected the
6442 // instructions to scalarize, there's nothing to do. Collection may already
6443 // have occurred if we have a user-selected VF and are now computing the
6444 // expected cost for interleaving.
6445 if (VF.isScalar() || VF.isZero() ||
6446 InstsToScalarize.find(VF) != InstsToScalarize.end())
6447 return;
6448
6449 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
6450 // not profitable to scalarize any instructions, the presence of VF in the
6451 // map will indicate that we've analyzed it already.
6452 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
6453
6454 // Find all the instructions that are scalar with predication in the loop and
6455 // determine if it would be better to not if-convert the blocks they are in.
6456 // If so, we also record the instructions to scalarize.
6457 for (BasicBlock *BB : TheLoop->blocks()) {
6458 if (!blockNeedsPredication(BB))
6459 continue;
6460 for (Instruction &I : *BB)
6461 if (isScalarWithPredication(&I)) {
6462 ScalarCostsTy ScalarCosts;
6463 // Do not apply discount logic if hacked cost is needed
6464 // for emulated masked memrefs.
6465 if (!useEmulatedMaskMemRefHack(&I) &&
6466 computePredInstDiscount(&I, ScalarCosts, VF) >= 0)
6467 ScalarCostsVF.insert(ScalarCosts.begin(), ScalarCosts.end());
6468 // Remember that BB will remain after vectorization.
6469 PredicatedBBsAfterVectorization.insert(BB);
6470 }
6471 }
6472}
6473
6474int LoopVectorizationCostModel::computePredInstDiscount(
6475 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
6476 assert(!isUniformAfterVectorization(PredInst, VF) &&
6477 "Instruction marked uniform-after-vectorization will be predicated");
6478
6479 // Initialize the discount to zero, meaning that the scalar version and the
6480 // vector version cost the same.
6481 InstructionCost Discount = 0;
6482
6483 // Holds instructions to analyze. The instructions we visit are mapped in
6484 // ScalarCosts. Those instructions are the ones that would be scalarized if
6485 // we find that the scalar version costs less.
6486 SmallVector<Instruction *, 8> Worklist;
6487
6488 // Returns true if the given instruction can be scalarized.
6489 auto canBeScalarized = [&](Instruction *I) -> bool {
6490 // We only attempt to scalarize instructions forming a single-use chain
6491 // from the original predicated block that would otherwise be vectorized.
6492 // Although not strictly necessary, we give up on instructions we know will
6493 // already be scalar to avoid traversing chains that are unlikely to be
6494 // beneficial.
6495 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
6496 isScalarAfterVectorization(I, VF))
6497 return false;
6498
6499 // If the instruction is scalar with predication, it will be analyzed
6500 // separately. We ignore it within the context of PredInst.
6501 if (isScalarWithPredication(I))
6502 return false;
6503
6504 // If any of the instruction's operands are uniform after vectorization,
6505 // the instruction cannot be scalarized. This prevents, for example, a
6506 // masked load from being scalarized.
6507 //
6508 // We assume we will only emit a value for lane zero of an instruction
6509 // marked uniform after vectorization, rather than VF identical values.
6510 // Thus, if we scalarize an instruction that uses a uniform, we would
6511 // create uses of values corresponding to the lanes we aren't emitting code
6512 // for. This behavior can be changed by allowing getScalarValue to clone
6513 // the lane zero values for uniforms rather than asserting.
6514 for (Use &U : I->operands())
6515 if (auto *J = dyn_cast<Instruction>(U.get()))
6516 if (isUniformAfterVectorization(J, VF))
6517 return false;
6518
6519 // Otherwise, we can scalarize the instruction.
6520 return true;
6521 };
6522
6523 // Compute the expected cost discount from scalarizing the entire expression
6524 // feeding the predicated instruction. We currently only consider expressions
6525 // that are single-use instruction chains.
6526 Worklist.push_back(PredInst);
6527 while (!Worklist.empty()) {
6528 Instruction *I = Worklist.pop_back_val();
6529
6530 // If we've already analyzed the instruction, there's nothing to do.
6531 if (ScalarCosts.find(I) != ScalarCosts.end())
6532 continue;
6533
6534 // Compute the cost of the vector instruction. Note that this cost already
6535 // includes the scalarization overhead of the predicated instruction.
6536 InstructionCost VectorCost = getInstructionCost(I, VF).first;
6537
6538 // Compute the cost of the scalarized instruction. This cost is the cost of
6539 // the instruction as if it wasn't if-converted and instead remained in the
6540 // predicated block. We will scale this cost by block probability after
6541 // computing the scalarization overhead.
6542 assert(!VF.isScalable() && "scalable vectors not yet supported.");
6543 InstructionCost ScalarCost =
6544 VF.getKnownMinValue() *
6545 getInstructionCost(I, ElementCount::getFixed(1)).first;
6546
6547 // Compute the scalarization overhead of needed insertelement instructions
6548 // and phi nodes.
6549 if (isScalarWithPredication(I) && !I->getType()->isVoidTy()) {
6550 ScalarCost += TTI.getScalarizationOverhead(
6551 cast<VectorType>(ToVectorTy(I->getType(), VF)),
6552 APInt::getAllOnesValue(VF.getKnownMinValue()), true, false);
6553 assert(!VF.isScalable() && "scalable vectors not yet supported.");
6554 ScalarCost +=
6555 VF.getKnownMinValue() *
6556 TTI.getCFInstrCost(Instruction::PHI, TTI::TCK_RecipThroughput);
6557 }
6558
6559 // Compute the scalarization overhead of needed extractelement
6560 // instructions. For each of the instruction's operands, if the operand can
6561 // be scalarized, add it to the worklist; otherwise, account for the
6562 // overhead.
6563 for (Use &U : I->operands())
6564 if (auto *J = dyn_cast<Instruction>(U.get())) {
6565 assert(VectorType::isValidElementType(J->getType()) &&
6566 "Instruction has non-scalar type");
6567 if (canBeScalarized(J))
6568 Worklist.push_back(J);
6569 else if (needsExtract(J, VF)) {
6570 assert(!VF.isScalable() && "scalable vectors not yet supported.");
6571 ScalarCost += TTI.getScalarizationOverhead(
6572 cast<VectorType>(ToVectorTy(J->getType(), VF)),
6573 APInt::getAllOnesValue(VF.getKnownMinValue()), false, true);
6574 }
6575 }
6576
6577 // Scale the total scalar cost by block probability.
6578 ScalarCost /= getReciprocalPredBlockProb();
6579
6580 // Compute the discount. A non-negative discount means the vector version
6581 // of the instruction costs more, and scalarizing would be beneficial.
6582 Discount += VectorCost - ScalarCost;
6583 ScalarCosts[I] = ScalarCost;
6584 }
6585
6586 return *Discount.getValue();
6587}
6588
6589LoopVectorizationCostModel::VectorizationCostTy
6590LoopVectorizationCostModel::expectedCost(ElementCount VF) {
6591 VectorizationCostTy Cost;
6592
6593 // For each block.
6594 for (BasicBlock *BB : TheLoop->blocks()) {
6595 VectorizationCostTy BlockCost;
6596
6597 // For each instruction in the old loop.
6598 for (Instruction &I : BB->instructionsWithoutDebug()) {
6599 // Skip ignored values.
6600 if (ValuesToIgnore.count(&I) ||
6601 (VF.isVector() && VecValuesToIgnore.count(&I)))
6602 continue;
6603
6604 VectorizationCostTy C = getInstructionCost(&I, VF);
6605
6606 // Check if we should override the cost.
6607 if (ForceTargetInstructionCost.getNumOccurrences() > 0)
6608 C.first = InstructionCost(ForceTargetInstructionCost);
6609
6610 BlockCost.first += C.first;
6611 BlockCost.second |= C.second;
6612 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C.first
6613 << " for VF " << VF << " For instruction: " << I
6614 << '\n');
6615 }
6616
6617 // If we are vectorizing a predicated block, it will have been
6618 // if-converted. This means that the block's instructions (aside from
6619 // stores and instructions that may divide by zero) will now be
6620 // unconditionally executed. For the scalar case, we may not always execute
6621 // the predicated block, if it is an if-else block. Thus, scale the block's
6622 // cost by the probability of executing it. blockNeedsPredication from
6623 // Legal is used so as to not include all blocks in tail folded loops.
6624 if (VF.isScalar() && Legal->blockNeedsPredication(BB))
6625 BlockCost.first /= getReciprocalPredBlockProb();
6626
6627 Cost.first += BlockCost.first;
6628 Cost.second |= BlockCost.second;
6629 }
6630
6631 return Cost;
6632}
6633
6634/// Gets Address Access SCEV after verifying that the access pattern
6635/// is loop invariant except the induction variable dependence.
6636///
6637/// This SCEV can be sent to the Target in order to estimate the address
6638/// calculation cost.
6639static const SCEV *getAddressAccessSCEV(
6640 Value *Ptr,
6641 LoopVectorizationLegality *Legal,
6642 PredicatedScalarEvolution &PSE,
6643 const Loop *TheLoop) {
6644
6645 auto *Gep = dyn_cast<GetElementPtrInst>(Ptr);
6646 if (!Gep)
6647 return nullptr;
6648
6649 // We are looking for a gep with all loop invariant indices except for one
6650 // which should be an induction variable.
6651 auto SE = PSE.getSE();
6652 unsigned NumOperands = Gep->getNumOperands();
6653 for (unsigned i = 1; i < NumOperands; ++i) {
6654 Value *Opd = Gep->getOperand(i);
6655 if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&
6656 !Legal->isInductionVariable(Opd))
6657 return nullptr;
6658 }
6659
6660 // Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV.
6661 return PSE.getSCEV(Ptr);
6662}
6663
6664static bool isStrideMul(Instruction *I, LoopVectorizationLegality *Legal) {
6665 return Legal->hasStride(I->getOperand(0)) ||
6666 Legal->hasStride(I->getOperand(1));
6667}
6668
6669InstructionCost
6670LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
6671 ElementCount VF) {
6672 assert(VF.isVector() &&
6673 "Scalarization cost of instruction implies vectorization.");
6674 assert(!VF.isScalable() && "scalable vectors not yet supported.");
6675 Type *ValTy = getMemInstValueType(I);
6676 auto SE = PSE.getSE();
6677
6678 unsigned AS = getLoadStoreAddressSpace(I);
6679 Value *Ptr = getLoadStorePointerOperand(I);
6680 Type *PtrTy = ToVectorTy(Ptr->getType(), VF);
6681
6682 // Figure out whether the access is strided and get the stride value
6683 // if it's known in compile time
6684 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop);
6685
6686 // Get the cost of the scalar memory instruction and address computation.
6687 InstructionCost Cost =
6688 VF.getKnownMinValue() * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV);
6689
6690 // Don't pass *I here, since it is scalar but will actually be part of a
6691 // vectorized loop where the user of it is a vectorized instruction.
6692 const Align Alignment = getLoadStoreAlignment(I);
6693 Cost += VF.getKnownMinValue() *
6694 TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
6695 AS, TTI::TCK_RecipThroughput);
6696
6697 // Get the overhead of the extractelement and insertelement instructions
6698 // we might create due to scalarization.
6699 Cost += getScalarizationOverhead(I, VF);
6700
6701 // If we have a predicated store, it may not be executed for each vector
6702 // lane. Scale the cost by the probability of executing the predicated
6703 // block.
6704 if (isPredicatedInst(I)) {
6705 Cost /= getReciprocalPredBlockProb();
6706
6707 if (useEmulatedMaskMemRefHack(I))
6708 // Artificially setting to a high enough value to practically disable
6709 // vectorization with such operations.
6710 Cost = 3000000;
6711 }
6712
6713 return Cost;
6714}
6715
6716InstructionCost
6717LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
6718 ElementCount VF) {
6719 Type *ValTy = getMemInstValueType(I);
6720 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
6721 Value *Ptr = getLoadStorePointerOperand(I);
6722 unsigned AS = getLoadStoreAddressSpace(I);
6723 int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
6724 enum TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
6725
6726 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
6727 "Stride should be 1 or -1 for consecutive memory access");
6728 const Align Alignment = getLoadStoreAlignment(I);
6729 InstructionCost Cost = 0;
6730 if (Legal->isMaskRequired(I))
6731 Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
6732 CostKind);
6733 else
6734 Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
6735 CostKind, I);
6736
6737 bool Reverse = ConsecutiveStride < 0;
6738 if (Reverse)
6739 Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);
6740 return Cost;
6741}
6742
6743InstructionCost
6744LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
6745 ElementCount VF) {
6746 assert(Legal->isUniformMemOp(*I));
6747
6748 Type *ValTy = getMemInstValueType(I);
6749 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
6750 const Align Alignment = getLoadStoreAlignment(I);
6751 unsigned AS = getLoadStoreAddressSpace(I);
6752 enum TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
6753 if (isa<LoadInst>(I)) {
6754 return TTI.getAddressComputationCost(ValTy) +
6755 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
6756 CostKind) +
6757 TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy);
6758 }
6759 StoreInst *SI = cast<StoreInst>(I);
6760
6761 bool isLoopInvariantStoreValue = Legal->isUniform(SI->getValueOperand());
6762 return TTI.getAddressComputationCost(ValTy) +
6763 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS,
6764 CostKind) +
6765 (isLoopInvariantStoreValue
6766 ? 0
6767 : TTI.getVectorInstrCost(Instruction::ExtractElement, VectorTy,
6768 VF.getKnownMinValue() - 1));
6769}
6770
6771InstructionCost
6772LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
6773 ElementCount VF) {
6774 Type *ValTy = getMemInstValueType(I);
6775 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
6776 const Align Alignment = getLoadStoreAlignment(I);
6777 const Value *Ptr = getLoadStorePointerOperand(I);
6778
6779 return TTI.getAddressComputationCost(VectorTy) +
6780 TTI.getGatherScatterOpCost(
6781 I->getOpcode(), VectorTy, Ptr, Legal->isMaskRequired(I), Alignment,
6782 TargetTransformInfo::TCK_RecipThroughput, I);
6783}
6784
6785InstructionCost
6786LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
6787 ElementCount VF) {
6788 Type *ValTy = getMemInstValueType(I);
6789 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
6790 unsigned AS = getLoadStoreAddressSpace(I);
6791
6792 auto Group = getInterleavedAccessGroup(I);
6793 assert(Group && "Fail to get an interleaved access group.");
6794
6795 unsigned InterleaveFactor = Group->getFactor();
6796 assert(!VF.isScalable() && "scalable vectors not yet supported.");
6797 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
6798
6799 // Holds the indices of existing members in an interleaved load group.
6800 // An interleaved store group doesn't need this as it doesn't allow gaps.
6801 SmallVector<unsigned, 4> Indices;
6802 if (isa<LoadInst>(I)) {
6803 for (unsigned i = 0; i < InterleaveFactor; i++)
6804 if (Group->getMember(i))
6805 Indices.push_back(i);
6806 }
6807
6808 // Calculate the cost of the whole interleaved group.
6809 bool UseMaskForGaps =
6810 Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed();
6811 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
6812 I->getOpcode(), WideVecTy, Group->getFactor(), Indices, Group->getAlign(),
6813 AS, TTI::TCK_RecipThroughput, Legal->isMaskRequired(I), UseMaskForGaps);
6814
6815 if (Group->isReverse()) {
6816 // TODO: Add support for reversed masked interleaved access.
6817 assert(!Legal->isMaskRequired(I) &&
6818 "Reverse masked interleaved access not supported.");
6819 Cost += Group->getNumMembers() *
6820 TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);
6821 }
6822 return Cost;
6823}
6824
6825InstructionCost LoopVectorizationCostModel::getReductionPatternCost(
6826 Instruction *I, ElementCount VF, Type *Ty, TTI::TargetCostKind CostKind) {
6827 // Early exit for no inloop reductions
6828 if (InLoopReductionChains.empty() || VF.isScalar() || !isa<VectorType>(Ty))
6829 return InstructionCost::getInvalid();
6830 auto *VectorTy = cast<VectorType>(Ty);
6831
6832 // We are looking for a pattern of, and finding the minimal acceptable cost:
6833 // reduce(mul(ext(A), ext(B))) or
6834 // reduce(mul(A, B)) or
6835 // reduce(ext(A)) or
6836 // reduce(A).
6837 // The basic idea is that we walk down the tree to do that, finding the root
6838 // reduction instruction in InLoopReductionImmediateChains. From there we find
6839 // the pattern of mul/ext and test the cost of the entire pattern vs the cost
6840 // of the components. If the reduction cost is lower then we return it for the
6841 // reduction instruction and 0 for the other instructions in the pattern. If
6842 // it is not we return an invalid cost specifying the orignal cost method
6843 // should be used.
6844 Instruction *RetI = I;
6845 if ((RetI->getOpcode() == Instruction::SExt ||
6846 RetI->getOpcode() == Instruction::ZExt)) {
6847 if (!RetI->hasOneUser())
6848 return InstructionCost::getInvalid();
6849 RetI = RetI->user_back();
6850 }
6851 if (RetI->getOpcode() == Instruction::Mul &&
6852 RetI->user_back()->getOpcode() == Instruction::Add) {
6853 if (!RetI->hasOneUser())
6854 return InstructionCost::getInvalid();
6855 RetI = RetI->user_back();
6856 }
6857
6858 // Test if the found instruction is a reduction, and if not return an invalid
6859 // cost specifying the parent to use the original cost modelling.
6860 if (!InLoopReductionImmediateChains.count(RetI))
6861 return InstructionCost::getInvalid();
6862
6863 // Find the reduction this chain is a part of and calculate the basic cost of
6864 // the reduction on its own.
6865 Instruction *LastChain = InLoopReductionImmediateChains[RetI];
6866 Instruction *ReductionPhi = LastChain;
6867 while (!isa<PHINode>(ReductionPhi))
6868 ReductionPhi = InLoopReductionImmediateChains[ReductionPhi];
6869
6870 RecurrenceDescriptor RdxDesc =
6871 Legal->getReductionVars()[cast<PHINode>(ReductionPhi)];
6872 unsigned BaseCost = TTI.getArithmeticReductionCost(RdxDesc.getOpcode(),
6873 VectorTy, false, CostKind);
6874
6875 // Get the operand that was not the reduction chain and match it to one of the
6876 // patterns, returning the better cost if it is found.
6877 Instruction *RedOp = RetI->getOperand(1) == LastChain
6878 ? dyn_cast<Instruction>(RetI->getOperand(0))
6879 : dyn_cast<Instruction>(RetI->getOperand(1));
6880
6881 VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy);
6882
6883 if (RedOp && (isa<SExtInst>(RedOp) || isa<ZExtInst>(RedOp)) &&
6884 !TheLoop->isLoopInvariant(RedOp)) {
6885 bool IsUnsigned = isa<ZExtInst>(RedOp);
6886 auto *ExtType = VectorType::get(RedOp->getOperand(0)->getType(), VectorTy);
6887 InstructionCost RedCost = TTI.getExtendedAddReductionCost(
6888 /*IsMLA=*/false, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
6889 CostKind);
6890
6891 unsigned ExtCost =
6892 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, ExtType,
6893 TTI::CastContextHint::None, CostKind, RedOp);
6894 if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
6895 return I == RetI ? *RedCost.getValue() : 0;
6896 } else if (RedOp && RedOp->getOpcode() == Instruction::Mul) {
6897 Instruction *Mul = RedOp;
6898 Instruction *Op0 = dyn_cast<Instruction>(Mul->getOperand(0));
6899 Instruction *Op1 = dyn_cast<Instruction>(Mul->getOperand(1));
6900 if (Op0 && Op1 && (isa<SExtInst>(Op0) || isa<ZExtInst>(Op0)) &&
6901 Op0->getOpcode() == Op1->getOpcode() &&
6902 Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
6903 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) {
6904 bool IsUnsigned = isa<ZExtInst>(Op0);
6905 auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy);
6906 // reduce(mul(ext, ext))
6907 unsigned ExtCost =
6908 TTI.getCastInstrCost(Op0->getOpcode(), VectorTy, ExtType,
6909 TTI::CastContextHint::None, CostKind, Op0);
6910 unsigned MulCost =
6911 TTI.getArithmeticInstrCost(Mul->getOpcode(), VectorTy, CostKind);
6912
6913 InstructionCost RedCost = TTI.getExtendedAddReductionCost(
6914 /*IsMLA=*/true, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
6915 CostKind);
6916
6917 if (RedCost.isValid() && RedCost < ExtCost * 2 + MulCost + BaseCost)
6918 return I == RetI ? *RedCost.getValue() : 0;
6919 } else {
6920 unsigned MulCost =
6921 TTI.getArithmeticInstrCost(Mul->getOpcode(), VectorTy, CostKind);
6922
6923 InstructionCost RedCost = TTI.getExtendedAddReductionCost(
6924 /*IsMLA=*/true, true, RdxDesc.getRecurrenceType(), VectorTy,
6925 CostKind);
6926
6927 if (RedCost.isValid() && RedCost < MulCost + BaseCost)
6928 return I == RetI ? *RedCost.getValue() : 0;
6929 }
6930 }
6931
6932 return I == RetI ? BaseCost : InstructionCost::getInvalid();
6933}
6934
6935InstructionCost
6936LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
6937 ElementCount VF) {
6938 // Calculate scalar cost only. Vectorization cost should be ready at this
6939 // moment.
6940 if (VF.isScalar()) {
6941 Type *ValTy = getMemInstValueType(I);
6942 const Align Alignment = getLoadStoreAlignment(I);
6943 unsigned AS = getLoadStoreAddressSpace(I);
6944
6945 return TTI.getAddressComputationCost(ValTy) +
6946 TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS,
6947 TTI::TCK_RecipThroughput, I);
6948 }
6949 return getWideningCost(I, VF);
6950}
6951
6952LoopVectorizationCostModel::VectorizationCostTy
6953LoopVectorizationCostModel::getInstructionCost(Instruction *I,
6954 ElementCount VF) {
6955 // If we know that this instruction will remain uniform, check the cost of
6956 // the scalar version.
6957 if (isUniformAfterVectorization(I, VF))
6958 VF = ElementCount::getFixed(1);
6959
6960 if (VF.isVector() && isProfitableToScalarize(I, VF))
6961 return VectorizationCostTy(InstsToScalarize[VF][I], false);
6962
6963 // Forced scalars do not have any scalarization overhead.
6964 auto ForcedScalar = ForcedScalars.find(VF);
6965 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
6966 auto InstSet = ForcedScalar->second;
6967 if (InstSet.count(I))
6968 return VectorizationCostTy(
6969 (getInstructionCost(I, ElementCount::getFixed(1)).first *
6970 VF.getKnownMinValue()),
6971 false);
6972 }
6973
6974 Type *VectorTy;
6975 InstructionCost C = getInstructionCost(I, VF, VectorTy);
6976
6977 bool TypeNotScalarized =
6978 VF.isVector() && VectorTy->isVectorTy() &&
6979 TTI.getNumberOfParts(VectorTy) < VF.getKnownMinValue();
6980 return VectorizationCostTy(C, TypeNotScalarized);
6981}
6982
6983InstructionCost
6984LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
6985 ElementCount VF) {
6986
6987 assert(!VF.isScalable() &&
6988 "cannot compute scalarization overhead for scalable vectorization");
6989 if (VF.isScalar())
6990 return 0;
6991
6992 InstructionCost Cost = 0;
6993 Type *RetTy = ToVectorTy(I->getType(), VF);
6994 if (!RetTy->isVoidTy() &&
6995 (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore()))
6996 Cost += TTI.getScalarizationOverhead(
6997 cast<VectorType>(RetTy), APInt::getAllOnesValue(VF.getKnownMinValue()),
6998 true, false);
6999
7000 // Some targets keep addresses scalar.
7001 if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing())
7002 return Cost;
7003
7004 // Some targets support efficient element stores.
7005 if (isa<StoreInst>(I) && TTI.supportsEfficientVectorElementLoadStore())
7006 return Cost;
7007
7008 // Collect operands to consider.
7009 CallInst *CI = dyn_cast<CallInst>(I);
7010 Instruction::op_range Ops = CI ? CI->arg_operands() : I->operands();
7011
7012 // Skip operands that do not require extraction/scalarization and do not incur
7013 // any overhead.
7014 return Cost + TTI.getOperandsScalarizationOverhead(
7015 filterExtractingOperands(Ops, VF), VF.getKnownMinValue());
7016}
7017
7018void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
7019 if (VF.isScalar())
7020 return;
7021 NumPredStores = 0;
7022 for (BasicBlock *BB : TheLoop->blocks()) {
7023 // For each instruction in the old loop.
7024 for (Instruction &I : *BB) {
7025 Value *Ptr = getLoadStorePointerOperand(&I);
7026 if (!Ptr)
7027 continue;
7028
7029 // TODO: We should generate better code and update the cost model for
7030 // predicated uniform stores. Today they are treated as any other
7031 // predicated store (see added test cases in
7032 // invariant-store-vectorization.ll).
7033 if (isa<StoreInst>(&I) && isScalarWithPredication(&I))
7034 NumPredStores++;
7035
7036 if (Legal->isUniformMemOp(I)) {
7037 // TODO: Avoid replicating loads and stores instead of
7038 // relying on instcombine to remove them.
7039 // Load: Scalar load + broadcast
7040 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
7041 InstructionCost Cost = getUniformMemOpCost(&I, VF);
7042 setWideningDecision(&I, VF, CM_Scalarize, Cost);
7043 continue;
7044 }
7045
7046 // We assume that widening is the best solution when possible.
7047 if (memoryInstructionCanBeWidened(&I, VF)) {
7048 InstructionCost Cost = getConsecutiveMemOpCost(&I, VF);
7049 int ConsecutiveStride =
7050 Legal->isConsecutivePtr(getLoadStorePointerOperand(&I));
7051 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
7052 "Expected consecutive stride.");
7053 InstWidening Decision =
7054 ConsecutiveStride == 1 ? CM_Widen : CM_Widen_Reverse;
7055 setWideningDecision(&I, VF, Decision, Cost);
7056 continue;
7057 }
7058
7059 // Choose between Interleaving, Gather/Scatter or Scalarization.
7060 InstructionCost InterleaveCost = std::numeric_limits<int>::max();
7061 unsigned NumAccesses = 1;
7062 if (isAccessInterleaved(&I)) {
7063 auto Group = getInterleavedAccessGroup(&I);
7064 assert(Group && "Fail to get an interleaved access group.");
7065
7066 // Make one decision for the whole group.
7067 if (getWideningDecision(&I, VF) != CM_Unknown)
7068 continue;
7069
7070 NumAccesses = Group->getNumMembers();
7071 if (interleavedAccessCanBeWidened(&I, VF))
7072 InterleaveCost = getInterleaveGroupCost(&I, VF);
7073 }
7074
7075 InstructionCost GatherScatterCost =
7076 isLegalGatherOrScatter(&I)
7077 ? getGatherScatterCost(&I, VF) * NumAccesses
7078 : std::numeric_limits<int>::max();
7079
7080 InstructionCost ScalarizationCost =
7081 getMemInstScalarizationCost(&I, VF) * NumAccesses;
7082
7083 // Choose better solution for the current VF,
7084 // write down this decision and use it during vectorization.
7085 InstructionCost Cost;
7086 InstWidening Decision;
7087 if (InterleaveCost <= GatherScatterCost &&
7088 InterleaveCost < ScalarizationCost) {
7089 Decision = CM_Interleave;
7090 Cost = InterleaveCost;
7091 } else if (GatherScatterCost < ScalarizationCost) {
7092 Decision = CM_GatherScatter;
7093 Cost = GatherScatterCost;
7094 } else {
7095 Decision = CM_Scalarize;
7096 Cost = ScalarizationCost;
7097 }
7098 // If the instructions belongs to an interleave group, the whole group
7099 // receives the same decision. The whole group receives the cost, but
7100 // the cost will actually be assigned to one instruction.
7101 if (auto Group = getInterleavedAccessGroup(&I))
7102 setWideningDecision(Group, VF, Decision, Cost);
7103 else
7104 setWideningDecision(&I, VF, Decision, Cost);
7105 }
7106 }
7107
7108 // Make sure that any load of address and any other address computation
7109 // remains scalar unless there is gather/scatter support. This avoids
7110 // inevitable extracts into address registers, and also has the benefit of
7111 // activating LSR more, since that pass can't optimize vectorized
7112 // addresses.
7113 if (TTI.prefersVectorizedAddressing())
7114 return;
7115
7116 // Start with all scalar pointer uses.
7117 SmallPtrSet<Instruction *, 8> AddrDefs;
7118 for (BasicBlock *BB : TheLoop->blocks())
7119 for (Instruction &I : *BB) {
7120 Instruction *PtrDef =
7121 dyn_cast_or_null<Instruction>(getLoadStorePointerOperand(&I));
7122 if (PtrDef && TheLoop->contains(PtrDef) &&
7123 getWideningDecision(&I, VF) != CM_GatherScatter)
7124 AddrDefs.insert(PtrDef);
7125 }
7126
7127 // Add all instructions used to generate the addresses.
7128 SmallVector<Instruction *, 4> Worklist;
7129 append_range(Worklist, AddrDefs);
7130 while (!Worklist.empty()) {
7131 Instruction *I = Worklist.pop_back_val();
7132 for (auto &Op : I->operands())
7133 if (auto *InstOp = dyn_cast<Instruction>(Op))
7134 if ((InstOp->getParent() == I->getParent()) && !isa<PHINode>(InstOp) &&
7135 AddrDefs.insert(InstOp).second)
7136 Worklist.push_back(InstOp);
7137 }
7138
7139 for (auto *I : AddrDefs) {
7140 if (isa<LoadInst>(I)) {
7141 // Setting the desired widening decision should ideally be handled in
7142 // by cost functions, but since this involves the task of finding out
7143 // if the loaded register is involved in an address computation, it is
7144 // instead changed here when we know this is the case.
7145 InstWidening Decision = getWideningDecision(I, VF);
7146 if (Decision == CM_Widen || Decision == CM_Widen_Reverse)
7147 // Scalarize a widened load of address.
7148 setWideningDecision(
7149 I, VF, CM_Scalarize,
7150 (VF.getKnownMinValue() *
7151 getMemoryInstructionCost(I, ElementCount::getFixed(1))));
7152 else if (auto Group = getInterleavedAccessGroup(I)) {
7153 // Scalarize an interleave group of address loads.
7154 for (unsigned I = 0; I < Group->getFactor(); ++I) {
7155 if (Instruction *Member = Group->getMember(I))
7156 setWideningDecision(
7157 Member, VF, CM_Scalarize,
7158 (VF.getKnownMinValue() *
7159 getMemoryInstructionCost(Member, ElementCount::getFixed(1))));
7160 }
7161 }
7162 } else
7163 // Make sure I gets scalarized and a cost estimate without
7164 // scalarization overhead.
7165 ForcedScalars[VF].insert(I);
7166 }
7167}
7168
7169InstructionCost
7170LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF,
7171 Type *&VectorTy) {
7172 Type *RetTy = I->getType();
7173 if (canTruncateToMinimalBitwidth(I, VF))
7174 RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
7175 VectorTy = isScalarAfterVectorization(I, VF) ? RetTy : ToVectorTy(RetTy, VF);
7176 auto SE = PSE.getSE();
7177 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
7178
7179 // TODO: We need to estimate the cost of intrinsic calls.
7180 switch (I->getOpcode()) {
7181 case Instruction::GetElementPtr:
7182 // We mark this instruction as zero-cost because the cost of GEPs in
7183 // vectorized code depends on whether the corresponding memory instruction
7184 // is scalarized or not. Therefore, we handle GEPs with the memory
7185 // instruction cost.
7186 return 0;
7187 case Instruction::Br: {
7188 // In cases of scalarized and predicated instructions, there will be VF
7189 // predicated blocks in the vectorized loop. Each branch around these
7190 // blocks requires also an extract of its vector compare i1 element.
7191 bool ScalarPredicatedBB = false;
7192 BranchInst *BI = cast<BranchInst>(I);
7193 if (VF.isVector() && BI->isConditional() &&
7194 (PredicatedBBsAfterVectorization.count(BI->getSuccessor(0)) ||
7195 PredicatedBBsAfterVectorization.count(BI->getSuccessor(1))))
7196 ScalarPredicatedBB = true;
7197
7198 if (ScalarPredicatedBB) {
7199 // Return cost for branches around scalarized and predicated blocks.
7200 assert(!VF.isScalable() && "scalable vectors not yet supported.");
7201 auto *Vec_i1Ty =
7202 VectorType::get(IntegerType::getInt1Ty(RetTy->getContext()), VF);
7203 return (TTI.getScalarizationOverhead(
7204 Vec_i1Ty, APInt::getAllOnesValue(VF.getKnownMinValue()),
7205 false, true) +
7206 (TTI.getCFInstrCost(Instruction::Br, CostKind) *
7207 VF.getKnownMinValue()));
7208 } else if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
7209 // The back-edge branch will remain, as will all scalar branches.
7210 return TTI.getCFInstrCost(Instruction::Br, CostKind);
7211 else
7212 // This branch will be eliminated by if-conversion.
7213 return 0;
7214 // Note: We currently assume zero cost for an unconditional branch inside
7215 // a predicated block since it will become a fall-through, although we
7216 // may decide in the future to call TTI for all branches.
7217 }
7218 case Instruction::PHI: {
7219 auto *Phi = cast<PHINode>(I);
7220
7221 // First-order recurrences are replaced by vector shuffles inside the loop.
7222 // NOTE: Don't use ToVectorTy as SK_ExtractSubvector expects a vector type.
7223 if (VF.isVector() && Legal->isFirstOrderRecurrence(Phi))
7224 return TTI.getShuffleCost(
7225 TargetTransformInfo::SK_ExtractSubvector, cast<VectorType>(VectorTy),
7226 VF.getKnownMinValue() - 1, FixedVectorType::get(RetTy, 1));
7227
7228 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
7229 // converted into select instructions. We require N - 1 selects per phi
7230 // node, where N is the number of incoming values.
7231 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader())
7232 return (Phi->getNumIncomingValues() - 1) *
7233 TTI.getCmpSelInstrCost(
7234 Instruction::Select, ToVectorTy(Phi->getType(), VF),
7235 ToVectorTy(Type::getInt1Ty(Phi->getContext()), VF),
7236 CmpInst::BAD_ICMP_PREDICATE, CostKind);
7237
7238 return TTI.getCFInstrCost(Instruction::PHI, CostKind);
7239 }
7240 case Instruction::UDiv:
7241 case Instruction::SDiv:
7242 case Instruction::URem:
7243 case Instruction::SRem:
7244 // If we have a predicated instruction, it may not be executed for each
7245 // vector lane. Get the scalarization cost and scale this amount by the
7246 // probability of executing the predicated block. If the instruction is not
7247 // predicated, we fall through to the next case.
7248 if (VF.isVector() && isScalarWithPredication(I)) {
7249 InstructionCost Cost = 0;
7250
7251 // These instructions have a non-void type, so account for the phi nodes
7252 // that we will create. This cost is likely to be zero. The phi node
7253 // cost, if any, should be scaled by the block probability because it
7254 // models a copy at the end of each predicated block.
7255 Cost += VF.getKnownMinValue() *
7256 TTI.getCFInstrCost(Instruction::PHI, CostKind);
7257
7258 // The cost of the non-predicated instruction.
7259 Cost += VF.getKnownMinValue() *
7260 TTI.getArithmeticInstrCost(I->getOpcode(), RetTy, CostKind);
7261
7262 // The cost of insertelement and extractelement instructions needed for
7263 // scalarization.
7264 Cost += getScalarizationOverhead(I, VF);
7265
7266 // Scale the cost by the probability of executing the predicated blocks.
7267 // This assumes the predicated block for each vector lane is equally
7268 // likely.
7269 return Cost / getReciprocalPredBlockProb();
7270 }
7271 LLVM_FALLTHROUGH;
7272 case Instruction::Add:
7273 case Instruction::FAdd:
7274 case Instruction::Sub:
7275 case Instruction::FSub:
7276 case Instruction::Mul:
7277 case Instruction::FMul:
7278 case Instruction::FDiv:
7279 case Instruction::FRem:
7280 case Instruction::Shl:
7281 case Instruction::LShr:
7282 case Instruction::AShr:
7283 case Instruction::And:
7284 case Instruction::Or:
7285 case Instruction::Xor: {
7286 // Since we will replace the stride by 1 the multiplication should go away.
7287 if (I->getOpcode() == Instruction::Mul && isStrideMul(I, Legal))
7288 return 0;
7289
7290 // Detect reduction patterns
7291 InstructionCost RedCost;
7292 if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind))
7293 .isValid())
7294 return RedCost;
7295
7296 // Certain instructions can be cheaper to vectorize if they have a constant
7297 // second vector operand. One example of this are shifts on x86.
7298 Value *Op2 = I->getOperand(1);
7299 TargetTransformInfo::OperandValueProperties Op2VP;
7300 TargetTransformInfo::OperandValueKind Op2VK =
7301 TTI.getOperandInfo(Op2, Op2VP);
7302 if (Op2VK == TargetTransformInfo::OK_AnyValue && Legal->isUniform(Op2))
7303 Op2VK = TargetTransformInfo::OK_UniformValue;
7304
7305 SmallVector<const Value *, 4> Operands(I->operand_values());
7306 unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1;
7307 return N * TTI.getArithmeticInstrCost(
7308 I->getOpcode(), VectorTy, CostKind,
7309 TargetTransformInfo::OK_AnyValue,
7310 Op2VK, TargetTransformInfo::OP_None, Op2VP, Operands, I);
7311 }
7312 case Instruction::FNeg: {
7313 assert(!VF.isScalable() && "VF is assumed to be non scalable.");
7314 unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1;
7315 return N * TTI.getArithmeticInstrCost(
7316 I->getOpcode(), VectorTy, CostKind,
7317 TargetTransformInfo::OK_AnyValue,
7318 TargetTransformInfo::OK_AnyValue,
7319 TargetTransformInfo::OP_None, TargetTransformInfo::OP_None,
7320 I->getOperand(0), I);
7321 }
7322 case Instruction::Select: {
7323 SelectInst *SI = cast<SelectInst>(I);
7324 const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
7325 bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
7326 Type *CondTy = SI->getCondition()->getType();
7327 if (!ScalarCond)
7328 CondTy = VectorType::get(CondTy, VF);
7329 return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy,
7330 CmpInst::BAD_ICMP_PREDICATE, CostKind, I);
7331 }
7332 case Instruction::ICmp:
7333 case Instruction::FCmp: {
7334 Type *ValTy = I->getOperand(0)->getType();
7335 Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
7336 if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))
7337 ValTy = IntegerType::get(ValTy->getContext(), MinBWs[Op0AsInstruction]);
7338 VectorTy = ToVectorTy(ValTy, VF);
7339 return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr,
7340 CmpInst::BAD_ICMP_PREDICATE, CostKind, I);
7341 }
7342 case Instruction::Store:
7343 case Instruction::Load: {
7344 ElementCount Width = VF;
7345 if (Width.isVector()) {
7346 InstWidening Decision = getWideningDecision(I, Width);
7347 assert(Decision != CM_Unknown &&
7348 "CM decision should be taken at this point");
7349 if (Decision == CM_Scalarize)
7350 Width = ElementCount::getFixed(1);
7351 }
7352 VectorTy = ToVectorTy(getMemInstValueType(I), Width);
7353 return getMemoryInstructionCost(I, VF);
7354 }
7355 case Instruction::ZExt:
7356 case Instruction::SExt:
7357 case Instruction::FPToUI:
7358 case Instruction::FPToSI:
7359 case Instruction::FPExt:
7360 case Instruction::PtrToInt:
7361 case Instruction::IntToPtr:
7362 case Instruction::SIToFP:
7363 case Instruction::UIToFP:
7364 case Instruction::Trunc:
7365 case Instruction::FPTrunc:
7366 case Instruction::BitCast: {
7367 // Computes the CastContextHint from a Load/Store instruction.
7368 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
7369 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
7370 "Expected a load or a store!");
7371
7372 if (VF.isScalar() || !TheLoop->contains(I))
7373 return TTI::CastContextHint::Normal;
7374
7375 switch (getWideningDecision(I, VF)) {
7376 case LoopVectorizationCostModel::CM_GatherScatter:
7377 return TTI::CastContextHint::GatherScatter;
7378 case LoopVectorizationCostModel::CM_Interleave:
7379 return TTI::CastContextHint::Interleave;
7380 case LoopVectorizationCostModel::CM_Scalarize:
7381 case LoopVectorizationCostModel::CM_Widen:
7382 return Legal->isMaskRequired(I) ? TTI::CastContextHint::Masked
7383 : TTI::CastContextHint::Normal;
7384 case LoopVectorizationCostModel::CM_Widen_Reverse:
7385 return TTI::CastContextHint::Reversed;
7386 case LoopVectorizationCostModel::CM_Unknown:
7387 llvm_unreachable("Instr did not go through cost modelling?");
7388 }
7389
7390 llvm_unreachable("Unhandled case!");
7391 };
7392
7393 unsigned Opcode = I->getOpcode();
7394 TTI::CastContextHint CCH = TTI::CastContextHint::None;
7395 // For Trunc, the context is the only user, which must be a StoreInst.
7396 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
7397 if (I->hasOneUse())
7398 if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin()))
7399 CCH = ComputeCCH(Store);
7400 }
7401 // For Z/Sext, the context is the operand, which must be a LoadInst.
7402 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
7403 Opcode == Instruction::FPExt) {
7404 if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0)))
7405 CCH = ComputeCCH(Load);
7406 }
7407
7408 // We optimize the truncation of induction variables having constant
7409 // integer steps. The cost of these truncations is the same as the scalar
7410 // operation.
7411 if (isOptimizableIVTruncate(I, VF)) {
7412 auto *Trunc = cast<TruncInst>(I);
7413 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
7414 Trunc->getSrcTy(), CCH, CostKind, Trunc);
7415 }
7416
7417 // Detect reduction patterns
7418 InstructionCost RedCost;
7419 if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind))
7420 .isValid())
7421 return RedCost;
7422
7423 Type *SrcScalarTy = I->getOperand(0)->getType();
7424 Type *SrcVecTy =
7425 VectorTy->isVectorTy() ? ToVectorTy(SrcScalarTy, VF) : SrcScalarTy;
7426 if (canTruncateToMinimalBitwidth(I, VF)) {
7427 // This cast is going to be shrunk. This may remove the cast or it might
7428 // turn it into slightly different cast. For example, if MinBW == 16,
7429 // "zext i8 %1 to i32" becomes "zext i8 %1 to i16".
7430 //
7431 // Calculate the modified src and dest types.
7432 Type *MinVecTy = VectorTy;
7433 if (Opcode == Instruction::Trunc) {
7434 SrcVecTy = smallestIntegerVectorType(SrcVecTy, MinVecTy);
7435 VectorTy =
7436 largestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);
7437 } else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) {
7438 SrcVecTy = largestIntegerVectorType(SrcVecTy, MinVecTy);
7439 VectorTy =
7440 smallestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);
7441 }
7442 }
7443
7444 assert(!VF.isScalable() && "VF is assumed to be non scalable");
7445 unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1;
7446 return N *
7447 TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH, CostKind, I);
7448 }
7449 case Instruction::Call: {
7450 bool NeedToScalarize;
7451 CallInst *CI = cast<CallInst>(I);
7452 InstructionCost CallCost = getVectorCallCost(CI, VF, NeedToScalarize);
7453 if (getVectorIntrinsicIDForCall(CI, TLI)) {
7454 InstructionCost IntrinsicCost = getVectorIntrinsicCost(CI, VF);
7455 return std::min(CallCost, IntrinsicCost);
7456 }
7457 return CallCost;
7458 }
7459 case Instruction::ExtractValue:
7460 return TTI.getInstructionCost(I, TTI::TCK_RecipThroughput);
7461 default:
7462 // The cost of executing VF copies of the scalar instruction. This opcode
7463 // is unknown. Assume that it is the same as 'mul'.
7464 return VF.getKnownMinValue() * TTI.getArithmeticInstrCost(
7465 Instruction::Mul, VectorTy, CostKind) +
7466 getScalarizationOverhead(I, VF);
7467 } // end of switch.
7468}
7469
7470char LoopVectorize::ID = 0;
7471
7472static const char lv_name[] = "Loop Vectorization";
7473
7474INITIALIZE_PASS_BEGIN(LoopVectorize, LV_NAME, lv_name, false, false)
7475INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
7476INITIALIZE_PASS_DEPENDENCY(BasicAAWrapperPass)
7477INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
7478INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
7479INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
7480INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
7481INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
7482INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
7483INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
7484INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)
7485INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)
7486INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
7487INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
7488INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy)
7489INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false)
7490
7491namespace llvm {
7492
7493Pass *createLoopVectorizePass() { return new LoopVectorize(); }
7494
7495Pass *createLoopVectorizePass(bool InterleaveOnlyWhenForced,
7496 bool VectorizeOnlyWhenForced) {
7497 return new LoopVectorize(InterleaveOnlyWhenForced, VectorizeOnlyWhenForced);
7498}
7499
7500} // end namespace llvm
7501
7502bool LoopVectorizationCostModel::isConsecutiveLoadOrStore(Instruction *Inst) {
7503 // Check if the pointer operand of a load or store instruction is
7504 // consecutive.
7505 if (auto *Ptr = getLoadStorePointerOperand(Inst))
7506 return Legal->isConsecutivePtr(Ptr);
7507 return false;
7508}
7509
7510void LoopVectorizationCostModel::collectValuesToIgnore() {
7511 // Ignore ephemeral values.
7512 CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore);
7513
7514 // Ignore type-promoting instructions we identified during reduction
7515 // detection.
7516 for (auto &Reduction : Legal->getReductionVars()) {
7517 RecurrenceDescriptor &RedDes = Reduction.second;
7518 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
7519 VecValuesToIgnore.insert(Casts.begin(), Casts.end());
7520 }
7521 // Ignore type-casting instructions we identified during induction
7522 // detection.
7523 for (auto &Induction : Legal->getInductionVars()) {
7524 InductionDescriptor &IndDes = Induction.second;
7525 const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
7526 VecValuesToIgnore.insert(Casts.begin(), Casts.end());
7527 }
7528}
7529
7530void LoopVectorizationCostModel::collectInLoopReductions() {
7531 for (auto &Reduction : Legal->getReductionVars()) {
7532 PHINode *Phi = Reduction.first;
7533 RecurrenceDescriptor &RdxDesc = Reduction.second;
7534
7535 // We don't collect reductions that are type promoted (yet).
7536 if (RdxDesc.getRecurrenceType() != Phi->getType())
7537 continue;
7538
7539 // If the target would prefer this reduction to happen "in-loop", then we
7540 // want to record it as such.
7541 unsigned Opcode = RdxDesc.getOpcode();
7542 if (!PreferInLoopReductions &&
7543 !TTI.preferInLoopReduction(Opcode, Phi->getType(),
7544 TargetTransformInfo::ReductionFlags()))
7545 continue;
7546
7547 // Check that we can correctly put the reductions into the loop, by
7548 // finding the chain of operations that leads from the phi to the loop
7549 // exit value.
7550 SmallVector<Instruction *, 4> ReductionOperations =
7551 RdxDesc.getReductionOpChain(Phi, TheLoop);
7552 bool InLoop = !ReductionOperations.empty();
7553 if (InLoop) {
7554 InLoopReductionChains[Phi] = ReductionOperations;
7555 // Add the elements to InLoopReductionImmediateChains for cost modelling.
7556 Instruction *LastChain = Phi;
7557 for (auto *I : ReductionOperations) {
7558 InLoopReductionImmediateChains[I] = LastChain;
7559 LastChain = I;
7560 }
7561 }
7562 LLVM_DEBUG(dbgs() << "LV: Using " << (InLoop ? "inloop" : "out of loop")
7563 << " reduction for phi: " << *Phi << "\n");
7564 }
7565}
7566
7567// TODO: we could return a pair of values that specify the max VF and
7568// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
7569// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
7570// doesn't have a cost model that can choose which plan to execute if
7571// more than one is generated.
7572static unsigned determineVPlanVF(const unsigned WidestVectorRegBits,
7573 LoopVectorizationCostModel &CM) {
7574 unsigned WidestType;
7575 std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
7576 return WidestVectorRegBits / WidestType;
7577}
7578
7579VectorizationFactor
7580LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
7581 assert(!UserVF.isScalable() && "scalable vectors not yet supported");
7582 ElementCount VF = UserVF;
7583 // Outer loop handling: They may require CFG and instruction level
7584 // transformations before even evaluating whether vectorization is profitable.
7585 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
7586 // the vectorization pipeline.
7587 if (!OrigLoop->isInnermost()) {
7588 // If the user doesn't provide a vectorization factor, determine a
7589 // reasonable one.
7590 if (UserVF.isZero()) {
7591 VF = ElementCount::getFixed(
7592 determineVPlanVF(TTI->getRegisterBitWidth(true /* Vector*/), CM));
7593 LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
7594
7595 // Make sure we have a VF > 1 for stress testing.
7596 if (VPlanBuildStressTest && (VF.isScalar() || VF.isZero())) {
7597 LLVM_DEBUG(dbgs() << "LV: VPlan stress testing: "
7598 << "overriding computed VF.\n");
7599 VF = ElementCount::getFixed(4);
7600 }
7601 }
7602 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
7603 assert(isPowerOf2_32(VF.getKnownMinValue()) &&
7604 "VF needs to be a power of two");
7605 LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "")
7606 << "VF " << VF << " to build VPlans.\n");
7607 buildVPlans(VF, VF);
7608
7609 // For VPlan build stress testing, we bail out after VPlan construction.
7610 if (VPlanBuildStressTest)
7611 return VectorizationFactor::Disabled();
7612
7613 return {VF, 0 /*Cost*/};
7614 }
7615
7616 LLVM_DEBUG(
7617 dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
7618 "VPlan-native path.\n");
7619 return VectorizationFactor::Disabled();
7620}
7621
7622Optional<VectorizationFactor>
7623LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
7624 assert(OrigLoop->isInnermost() && "Inner loop expected.");
7625 Optional<ElementCount> MaybeMaxVF = CM.computeMaxVF(UserVF, UserIC);
7626 if (!MaybeMaxVF) // Cases that should not to be vectorized nor interleaved.
7627 return None;
7628
7629 // Invalidate interleave groups if all blocks of loop will be predicated.
7630 if (CM.blockNeedsPredication(OrigLoop->getHeader()) &&
7631 !useMaskedInterleavedAccesses(*TTI)) {
7632 LLVM_DEBUG(
7633 dbgs()
7634 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
7635 "which requires masked-interleaved support.\n");
7636 if (CM.InterleaveInfo.invalidateGroups())
7637 // Invalidating interleave groups also requires invalidating all decisions
7638 // based on them, which includes widening decisions and uniform and scalar
7639 // values.
7640 CM.invalidateCostModelingDecisions();
7641 }
7642
7643 ElementCount MaxVF = MaybeMaxVF.getValue();
7644 assert(MaxVF.isNonZero() && "MaxVF is zero.");
7645
7646 bool UserVFIsLegal = ElementCount::isKnownLE(UserVF, MaxVF);
7647 if (!UserVF.isZero() &&
7648 (UserVFIsLegal || (UserVF.isScalable() && MaxVF.isScalable()))) {
7649 // FIXME: MaxVF is temporarily used inplace of UserVF for illegal scalable
7650 // VFs here, this should be reverted to only use legal UserVFs once the
7651 // loop below supports scalable VFs.
7652 ElementCount VF = UserVFIsLegal ? UserVF : MaxVF;
7653 LLVM_DEBUG(dbgs() << "LV: Using " << (UserVFIsLegal ? "user" : "max")
7654 << " VF " << VF << ".\n");
7655 assert(isPowerOf2_32(VF.getKnownMinValue()) &&
7656 "VF needs to be a power of two");
7657 // Collect the instructions (and their associated costs) that will be more
7658 // profitable to scalarize.
7659 CM.selectUserVectorizationFactor(VF);
7660 CM.collectInLoopReductions();
7661 buildVPlansWithVPRecipes(VF, VF);
7662 LLVM_DEBUG(printPlans(dbgs()));
7663 return {{VF, 0}};
7664 }
7665
7666 assert(!MaxVF.isScalable() &&
7667 "Scalable vectors not yet supported beyond this point");
7668
7669 for (ElementCount VF = ElementCount::getFixed(1);
7670 ElementCount::isKnownLE(VF, MaxVF); VF *= 2) {
7671 // Collect Uniform and Scalar instructions after vectorization with VF.
7672 CM.collectUniformsAndScalars(VF);
7673
7674 // Collect the instructions (and their associated costs) that will be more
7675 // profitable to scalarize.
7676 if (VF.isVector())
7677 CM.collectInstsToScalarize(VF);
7678 }
7679
7680 CM.collectInLoopReductions();
7681
7682 buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxVF);
7683 LLVM_DEBUG(printPlans(dbgs()));
7684 if (MaxVF.isScalar())
7685 return VectorizationFactor::Disabled();
7686
7687 // Select the optimal vectorization factor.
7688 return CM.selectVectorizationFactor(MaxVF);
7689}
7690
7691void LoopVectorizationPlanner::setBestPlan(ElementCount VF, unsigned UF) {
7692 LLVM_DEBUG(dbgs() << "Setting best plan to VF=" << VF << ", UF=" << UF
7693 << '\n');
7694 BestVF = VF;
7695 BestUF = UF;
7696
7697 erase_if(VPlans, [VF](const VPlanPtr &Plan) {
7698 return !Plan->hasVF(VF);
7699 });
7700 assert(VPlans.size() == 1 && "Best VF has not a single VPlan.");
7701}
7702
7703void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV,
7704 DominatorTree *DT) {
7705 // Perform the actual loop transformation.
7706
7707 // 1. Create a new empty loop. Unlink the old loop and connect the new one.
7708 VPCallbackILV CallbackILV(ILV);
7709
7710 assert(BestVF.hasValue() && "Vectorization Factor is missing");
7711
7712 VPTransformState State{*BestVF,
7713 BestUF,
7714 OrigLoop,
7715 LI,
7716 DT,
7717 ILV.Builder,
7718 ILV.VectorLoopValueMap,
7719 &ILV,
7720 CallbackILV};
7721 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
7722 State.TripCount = ILV.getOrCreateTripCount(nullptr);
7723 State.CanonicalIV = ILV.Induction;
7724
7725 ILV.printDebugTracesAtStart();
7726
7727 //===------------------------------------------------===//
7728 //
7729 // Notice: any optimization or new instruction that go
7730 // into the code below should also be implemented in
7731 // the cost-model.
7732 //
7733 //===------------------------------------------------===//
7734
7735 // 2. Copy and widen instructions from the old loop into the new loop.
7736 assert(VPlans.size() == 1 && "Not a single VPlan to execute.");
7737 VPlans.front()->execute(&State);
7738
7739 // 3. Fix the vectorized code: take care of header phi's, live-outs,
7740 // predication, updating analyses.
7741 ILV.fixVectorizedLoop();
7742
7743 ILV.printDebugTracesAtEnd();
7744}
7745
7746void LoopVectorizationPlanner::collectTriviallyDeadInstructions(
7747 SmallPtrSetImpl<Instruction *> &DeadInstructions) {
7748
7749 // We create new control-flow for the vectorized loop, so the original exit
7750 // conditions will be dead after vectorization if it's only used by the
7751 // terminator
7752 SmallVector<BasicBlock*> ExitingBlocks;
7753 OrigLoop->getExitingBlocks(ExitingBlocks);
7754 for (auto *BB : ExitingBlocks) {
7755 auto *Cmp = dyn_cast<Instruction>(BB->getTerminator()->getOperand(0));
7756 if (!Cmp || !Cmp->hasOneUse())
7757 continue;
7758
7759 // TODO: we should introduce a getUniqueExitingBlocks on Loop
7760 if (!DeadInstructions.insert(Cmp).second)
7761 continue;
7762
7763 // The operands of the icmp is often a dead trunc, used by IndUpdate.
7764 // TODO: can recurse through operands in general
7765 for (Value *Op : Cmp->operands()) {
7766 if (isa<TruncInst>(Op) && Op->hasOneUse())
7767 DeadInstructions.insert(cast<Instruction>(Op));
7768 }
7769 }
7770
7771 // We create new "steps" for induction variable updates to which the original
7772 // induction variables map. An original update instruction will be dead if
7773 // all its users except the induction variable are dead.
7774 auto *Latch = OrigLoop->getLoopLatch();
7775 for (auto &Induction : Legal->getInductionVars()) {
7776 PHINode *Ind = Induction.first;
7777 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
7778
7779 // If the tail is to be folded by masking, the primary induction variable,
7780 // if exists, isn't dead: it will be used for masking. Don't kill it.
7781 if (CM.foldTailByMasking() && IndUpdate == Legal->getPrimaryInduction())
7782 continue;
7783
7784 if (llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
7785 return U == Ind || DeadInstructions.count(cast<Instruction>(U));
7786 }))
7787 DeadInstructions.insert(IndUpdate);
7788
7789 // We record as "Dead" also the type-casting instructions we had identified
7790 // during induction analysis. We don't need any handling for them in the
7791 // vectorized loop because we have proven that, under a proper runtime
7792 // test guarding the vectorized loop, the value of the phi, and the casted
7793 // value of the phi, are the same. The last instruction in this casting chain
7794 // will get its scalar/vector/widened def from the scalar/vector/widened def
7795 // of the respective phi node. Any other casts in the induction def-use chain
7796 // have no other uses outside the phi update chain, and will be ignored.
7797 InductionDescriptor &IndDes = Induction.second;
7798 const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
7799 DeadInstructions.insert(Casts.begin(), Casts.end());
7800 }
7801}
7802
7803Value *InnerLoopUnroller::reverseVector(Value *Vec) { return Vec; }
7804
7805Value *InnerLoopUnroller::getBroadcastInstrs(Value *V) { return V; }
7806
7807Value *InnerLoopUnroller::getStepVector(Value *Val, int StartIdx, Value *Step,
7808 Instruction::BinaryOps BinOp) {
7809 // When unrolling and the VF is 1, we only need to add a simple scalar.
7810 Type *Ty = Val->getType();
7811 assert(!Ty->isVectorTy() && "Val must be a scalar");
7812
7813 if (Ty->isFloatingPointTy()) {
7814 Constant *C = ConstantFP::get(Ty, (double)StartIdx);
7815
7816 // Floating point operations had to be 'fast' to enable the unrolling.
7817 Value *MulOp = addFastMathFlag(Builder.CreateFMul(C, Step));
7818 return addFastMathFlag(Builder.CreateBinOp(BinOp, Val, MulOp));
7819 }
7820 Constant *C = ConstantInt::get(Ty, StartIdx);
7821 return Builder.CreateAdd(Val, Builder.CreateMul(C, Step), "induction");
7822}
7823
7824static void AddRuntimeUnrollDisableMetaData(Loop *L) {
7825 SmallVector<Metadata *, 4> MDs;
7826 // Reserve first location for self reference to the LoopID metadata node.
7827 MDs.push_back(nullptr);
7828 bool IsUnrollMetadata = false;
7829 MDNode *LoopID = L->getLoopID();
7830 if (LoopID) {
7831 // First find existing loop unrolling disable metadata.
7832 for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
7833 auto *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
7834 if (MD) {
7835 const auto *S = dyn_cast<MDString>(MD->getOperand(0));
7836 IsUnrollMetadata =
7837 S && S->getString().startswith("llvm.loop.unroll.disable");
7838 }
7839 MDs.push_back(LoopID->getOperand(i));
7840 }
7841 }
7842
7843 if (!IsUnrollMetadata) {
7844 // Add runtime unroll disable metadata.
7845 LLVMContext &Context = L->getHeader()->getContext();
7846 SmallVector<Metadata *, 1> DisableOperands;
7847 DisableOperands.push_back(
7848 MDString::get(Context, "llvm.loop.unroll.runtime.disable"));
7849 MDNode *DisableNode = MDNode::get(Context, DisableOperands);
7850 MDs.push_back(DisableNode);
7851 MDNode *NewLoopID = MDNode::get(Context, MDs);
7852 // Set operand 0 to refer to the loop id itself.
7853 NewLoopID->replaceOperandWith(0, NewLoopID);
7854 L->setLoopID(NewLoopID);
7855 }
7856}
7857
7858//===--------------------------------------------------------------------===//
7859// EpilogueVectorizerMainLoop
7860//===--------------------------------------------------------------------===//
7861
7862/// This function is partially responsible for generating the control flow
7863/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
7864BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() {
7865 MDNode *OrigLoopID = OrigLoop->getLoopID();
7866 Loop *Lp = createVectorLoopSkeleton("");
7867
7868 // Generate the code to check the minimum iteration count of the vector
7869 // epilogue (see below).
7870 EPI.EpilogueIterationCountCheck =
7871 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, true);
7872 EPI.EpilogueIterationCountCheck->setName("iter.check");
7873
7874 // Generate the code to check any assumptions that we've made for SCEV
7875 // expressions.
7876 BasicBlock *SavedPreHeader = LoopVectorPreHeader;
7877 emitSCEVChecks(Lp, LoopScalarPreHeader);
7878
7879 // If a safety check was generated save it.
7880 if (SavedPreHeader != LoopVectorPreHeader)
7881 EPI.SCEVSafetyCheck = SavedPreHeader;
7882
7883 // Generate the code that checks at runtime if arrays overlap. We put the
7884 // checks into a separate block to make the more common case of few elements
7885 // faster.
7886 SavedPreHeader = LoopVectorPreHeader;
7887 emitMemRuntimeChecks(Lp, LoopScalarPreHeader);
7888
7889 // If a safety check was generated save/overwite it.
7890 if (SavedPreHeader != LoopVectorPreHeader)
7891 EPI.MemSafetyCheck = SavedPreHeader;
7892
7893 // Generate the iteration count check for the main loop, *after* the check
7894 // for the epilogue loop, so that the path-length is shorter for the case
7895 // that goes directly through the vector epilogue. The longer-path length for
7896 // the main loop is compensated for, by the gain from vectorizing the larger
7897 // trip count. Note: the branch will get updated later on when we vectorize
7898 // the epilogue.
7899 EPI.MainLoopIterationCountCheck =
7900 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, false);
7901
7902 // Generate the induction variable.
7903 OldInduction = Legal->getPrimaryInduction();
7904 Type *IdxTy = Legal->getWidestInductionType();
7905 Value *StartIdx = ConstantInt::get(IdxTy, 0);
7906 Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
7907 Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
7908 EPI.VectorTripCount = CountRoundDown;
7909 Induction =
7910 createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
7911 getDebugLocFromInstOrOperands(OldInduction));
7912
7913 // Skip induction resume value creation here because they will be created in
7914 // the second pass. If we created them here, they wouldn't be used anyway,
7915 // because the vplan in the second pass still contains the inductions from the
7916 // original loop.
7917
7918 return completeLoopSkeleton(Lp, OrigLoopID);
7919}
7920
7921void EpilogueVectorizerMainLoop::printDebugTracesAtStart() {
7922 LLVM_DEBUG({
7923 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
7924 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue()
7925 << ", Main Loop UF:" << EPI.MainLoopUF
7926 << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue()
7927 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
7928 });
7929}
7930
7931void EpilogueVectorizerMainLoop::printDebugTracesAtEnd() {
7932 DEBUG_WITH_TYPE(VerboseDebug, {
7933 dbgs() << "intermediate fn:\n" << *Induction->getFunction() << "\n";
7934 });
7935}
7936
7937BasicBlock *EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(
7938 Loop *L, BasicBlock *Bypass, bool ForEpilogue) {
7939 assert(L && "Expected valid Loop.");
7940 assert(Bypass && "Expected valid bypass basic block.");
7941 unsigned VFactor =
7942 ForEpilogue ? EPI.EpilogueVF.getKnownMinValue() : VF.getKnownMinValue();
7943 unsigned UFactor = ForEpilogue ? EPI.EpilogueUF : UF;
7944 Value *Count = getOrCreateTripCount(L);
7945 // Reuse existing vector loop preheader for TC checks.
7946 // Note that new preheader block is generated for vector loop.
7947 BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
7948 IRBuilder<> Builder(TCCheckBlock->getTerminator());
7949
7950 // Generate code to check if the loop's trip count is less than VF * UF of the
7951 // main vector loop.
7952 auto P =
7953 Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;
7954
7955 Value *CheckMinIters = Builder.CreateICmp(
7956 P, Count, ConstantInt::get(Count->getType(), VFactor * UFactor),
7957 "min.iters.check");
7958
7959 if (!ForEpilogue)
7960 TCCheckBlock->setName("vector.main.loop.iter.check");
7961
7962 // Create new preheader for vector loop.
7963 LoopVectorPreHeader = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
7964 DT, LI, nullptr, "vector.ph");
7965
7966 if (ForEpilogue) {
7967 assert(DT->properlyDominates(DT->getNode(TCCheckBlock),
7968 DT->getNode(Bypass)->getIDom()) &&
7969 "TC check is expected to dominate Bypass");
7970
7971 // Update dominator for Bypass & LoopExit.
7972 DT->changeImmediateDominator(Bypass, TCCheckBlock);
7973 DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);
7974
7975 LoopBypassBlocks.push_back(TCCheckBlock);
7976
7977 // Save the trip count so we don't have to regenerate it in the
7978 // vec.epilog.iter.check. This is safe to do because the trip count
7979 // generated here dominates the vector epilog iter check.
7980 EPI.TripCount = Count;
7981 }
7982
7983 ReplaceInstWithInst(
7984 TCCheckBlock->getTerminator(),
7985 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
7986
7987 return TCCheckBlock;
7988}
7989
7990//===--------------------------------------------------------------------===//
7991// EpilogueVectorizerEpilogueLoop
7992//===--------------------------------------------------------------------===//
7993
7994/// This function is partially responsible for generating the control flow
7995/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
7996BasicBlock *
7997EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
7998 MDNode *OrigLoopID = OrigLoop->getLoopID();
7999 Loop *Lp = createVectorLoopSkeleton("vec.epilog.");
8000
8001 // Now, compare the remaining count and if there aren't enough iterations to
8002 // execute the vectorized epilogue skip to the scalar part.
8003 BasicBlock *VecEpilogueIterationCountCheck = LoopVectorPreHeader;
8004 VecEpilogueIterationCountCheck->setName("vec.epilog.iter.check");
8005 LoopVectorPreHeader =
8006 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
8007 LI, nullptr, "vec.epilog.ph");
8008 emitMinimumVectorEpilogueIterCountCheck(Lp, LoopScalarPreHeader,
8009 VecEpilogueIterationCountCheck);
8010
8011 // Adjust the control flow taking the state info from the main loop
8012 // vectorization into account.
8013 assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
8014 "expected this to be saved from the previous pass.");
8015 EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith(
8016 VecEpilogueIterationCountCheck, LoopVectorPreHeader);
8017
8018 DT->changeImmediateDominator(LoopVectorPreHeader,
8019 EPI.MainLoopIterationCountCheck);
8020
8021 EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
8022 VecEpilogueIterationCountCheck, LoopScalarPreHeader);
8023
8024 if (EPI.SCEVSafetyCheck)
8025 EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith(
8026 VecEpilogueIterationCountCheck, LoopScalarPreHeader);
8027 if (EPI.MemSafetyCheck)
8028 EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith(
8029 VecEpilogueIterationCountCheck, LoopScalarPreHeader);
8030
8031 DT->changeImmediateDominator(
8032 VecEpilogueIterationCountCheck,
8033 VecEpilogueIterationCountCheck->getSinglePredecessor());
8034
8035 DT->changeImmediateDominator(LoopScalarPreHeader,
8036 EPI.EpilogueIterationCountCheck);
8037 DT->changeImmediateDominator(LoopExitBlock, EPI.EpilogueIterationCountCheck);
8038
8039 // Keep track of bypass blocks, as they feed start values to the induction
8040 // phis in the scalar loop preheader.
8041 if (EPI.SCEVSafetyCheck)
8042 LoopBypassBlocks.push_back(EPI.SCEVSafetyCheck);
8043 if (EPI.MemSafetyCheck)
8044 LoopBypassBlocks.push_back(EPI.MemSafetyCheck);
8045 LoopBypassBlocks.push_back(EPI.EpilogueIterationCountCheck);
8046
8047 // Generate a resume induction for the vector epilogue and put it in the
8048 // vector epilogue preheader
8049 Type *IdxTy = Legal->getWidestInductionType();
8050 PHINode *EPResumeVal = PHINode::Create(IdxTy, 2, "vec.epilog.resume.val",
8051 LoopVectorPreHeader->getFirstNonPHI());
8052 EPResumeVal->addIncoming(EPI.VectorTripCount, VecEpilogueIterationCountCheck);
8053 EPResumeVal->addIncoming(ConstantInt::get(IdxTy, 0),
8054 EPI.MainLoopIterationCountCheck);
8055
8056 // Generate the induction variable.
8057 OldInduction = Legal->getPrimaryInduction();
8058 Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
8059 Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
8060 Value *StartIdx = EPResumeVal;
8061 Induction =
8062 createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
8063 getDebugLocFromInstOrOperands(OldInduction));
8064
8065 // Generate induction resume values. These variables save the new starting
8066 // indexes for the scalar loop. They are used to test if there are any tail
8067 // iterations left once the vector loop has completed.
8068 // Note that when the vectorized epilogue is skipped due to iteration count
8069 // check, then the resume value for the induction variable comes from
8070 // the trip count of the main vector loop, hence passing the AdditionalBypass
8071 // argument.
8072 createInductionResumeValues(Lp, CountRoundDown,
8073 {VecEpilogueIterationCountCheck,
8074 EPI.VectorTripCount} /* AdditionalBypass */);
8075
8076 AddRuntimeUnrollDisableMetaData(Lp);
8077 return completeLoopSkeleton(Lp, OrigLoopID);
8078}
8079
8080BasicBlock *
8081EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
8082 Loop *L, BasicBlock *Bypass, BasicBlock *Insert) {
8083
8084 assert(EPI.TripCount &&
8085 "Expected trip count to have been safed in the first pass.");
8086 assert(
8087 (!isa<Instruction>(EPI.TripCount) ||
8088 DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) &&
8089 "saved trip count does not dominate insertion point.");
8090 Value *TC = EPI.TripCount;
8091 IRBuilder<> Builder(Insert->getTerminator());
8092 Value *Count = Builder.CreateSub(TC, EPI.VectorTripCount, "n.vec.remaining");
8093
8094 // Generate code to check if the loop's trip count is less than VF * UF of the
8095 // vector epilogue loop.
8096 auto P =
8097 Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;
8098
8099 Value *CheckMinIters = Builder.CreateICmp(
8100 P, Count,
8101 ConstantInt::get(Count->getType(),
8102 EPI.EpilogueVF.getKnownMinValue() * EPI.EpilogueUF),
8103 "min.epilog.iters.check");
8104
8105 ReplaceInstWithInst(
8106 Insert->getTerminator(),
8107 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
8108
8109 LoopBypassBlocks.push_back(Insert);
8110 return Insert;
8111}
8112
8113void EpilogueVectorizerEpilogueLoop::printDebugTracesAtStart() {
8114 LLVM_DEBUG({
8115 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
8116 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue()
8117 << ", Main Loop UF:" << EPI.MainLoopUF
8118 << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue()
8119 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
8120 });
8121}
8122
8123void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
8124 DEBUG_WITH_TYPE(VerboseDebug, {
8125 dbgs() << "final fn:\n" << *Induction->getFunction() << "\n";
8126 });
8127}
8128
8129bool LoopVectorizationPlanner::getDecisionAndClampRange(
8130 const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
8131 assert(!Range.isEmpty() && "Trying to test an empty VF range.");
8132 bool PredicateAtRangeStart = Predicate(Range.Start);
8133
8134 for (ElementCount TmpVF = Range.Start * 2;
8135 ElementCount::isKnownLT(TmpVF, Range.End); TmpVF *= 2)
8136 if (Predicate(TmpVF) != PredicateAtRangeStart) {
8137 Range.End = TmpVF;
8138 break;
8139 }
8140
8141 return PredicateAtRangeStart;
8142}
8143
8144/// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF,
8145/// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range
8146/// of VF's starting at a given VF and extending it as much as possible. Each
8147/// vectorization decision can potentially shorten this sub-range during
8148/// buildVPlan().
8149void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
8150 ElementCount MaxVF) {
8151 auto MaxVFPlusOne = MaxVF.getWithIncrement(1);
8152 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
8153 VFRange SubRange = {VF, MaxVFPlusOne};
8154 VPlans.push_back(buildVPlan(SubRange));
8155 VF = SubRange.End;
8156 }
8157}
8158
8159VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
8160 VPlanPtr &Plan) {
8161 assert(is_contained(predecessors(Dst), Src) && "Invalid edge");
8162
8163 // Look for cached value.
8164 std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);
8165 EdgeMaskCacheTy::iterator ECEntryIt = EdgeMaskCache.find(Edge);
8166 if (ECEntryIt != EdgeMaskCache.end())
8167 return ECEntryIt->second;
8168
8169 VPValue *SrcMask = createBlockInMask(Src, Plan);
8170
8171 // The terminator has to be a branch inst!
8172 BranchInst *BI = dyn_cast<BranchInst>(Src->getTerminator());
8173 assert(BI && "Unexpected terminator found");
8174
8175 if (!BI->isConditional() || BI->getSuccessor(0) == BI->getSuccessor(1))
8176 return EdgeMaskCache[Edge] = SrcMask;
8177
8178 // If source is an exiting block, we know the exit edge is dynamically dead
8179 // in the vector loop, and thus we don't need to restrict the mask. Avoid
8180 // adding uses of an otherwise potentially dead instruction.
8181 if (OrigLoop->isLoopExiting(Src))
8182 return EdgeMaskCache[Edge] = SrcMask;
8183
8184 VPValue *EdgeMask = Plan->getOrAddVPValue(BI->getCondition());
8185 assert(EdgeMask && "No Edge Mask found for condition");
8186
8187 if (BI->getSuccessor(0) != Dst)
8188 EdgeMask = Builder.createNot(EdgeMask);
8189
8190 if (SrcMask) { // Otherwise block in-mask is all-one, no need to AND.
8191 // The condition is 'SrcMask && EdgeMask', which is equivalent to
8192 // 'select i1 SrcMask, i1 EdgeMask, i1 false'.
8193 // The select version does not introduce new UB if SrcMask is false and
8194 // EdgeMask is poison. Using 'and' here introduces undefined behavior.
8195 VPValue *False = Plan->getOrAddVPValue(
8196 ConstantInt::getFalse(BI->getCondition()->getType()));
8197 EdgeMask = Builder.createSelect(SrcMask, EdgeMask, False);
8198 }
8199
8200 return EdgeMaskCache[Edge] = EdgeMask;
8201}
8202
8203VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
8204 assert(OrigLoop->contains(BB) && "Block is not a part of a loop");
8205
8206 // Look for cached value.
8207 BlockMaskCacheTy::iterator BCEntryIt = BlockMaskCache.find(BB);
8208 if (BCEntryIt != BlockMaskCache.end())
8209 return BCEntryIt->second;
8210
8211 // All-one mask is modelled as no-mask following the convention for masked
8212 // load/store/gather/scatter. Initialize BlockMask to no-mask.
8213 VPValue *BlockMask = nullptr;
8214
8215 if (OrigLoop->getHeader() == BB) {
8216 if (!CM.blockNeedsPredication(BB))
8217 return BlockMaskCache[BB] = BlockMask; // Loop incoming mask is all-one.
8218
8219 // Create the block in mask as the first non-phi instruction in the block.
8220 VPBuilder::InsertPointGuard Guard(Builder);
8221 auto NewInsertionPoint = Builder.getInsertBlock()->getFirstNonPhi();
8222 Builder.setInsertPoint(Builder.getInsertBlock(), NewInsertionPoint);
8223
8224 // Introduce the early-exit compare IV <= BTC to form header block mask.
8225 // This is used instead of IV < TC because TC may wrap, unlike BTC.
8226 // Start by constructing the desired canonical IV.
8227 VPValue *IV = nullptr;
8228 if (Legal->getPrimaryInduction())
8229 IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction());
8230 else {
8231 auto IVRecipe = new VPWidenCanonicalIVRecipe();
8232 Builder.getInsertBlock()->insert(IVRecipe, NewInsertionPoint);
8233 IV = IVRecipe->getVPValue();
8234 }
8235 VPValue *BTC = Plan->getOrCreateBackedgeTakenCount();
8236 bool TailFolded = !CM.isScalarEpilogueAllowed();
8237
8238 if (TailFolded && CM.TTI.emitGetActiveLaneMask()) {
8239 // While ActiveLaneMask is a binary op that consumes the loop tripcount
8240 // as a second argument, we only pass the IV here and extract the
8241 // tripcount from the transform state where codegen of the VP instructions
8242 // happen.
8243 BlockMask = Builder.createNaryOp(VPInstruction::ActiveLaneMask, {IV});
8244 } else {
8245 BlockMask = Builder.createNaryOp(VPInstruction::ICmpULE, {IV, BTC});
8246 }
8247 return BlockMaskCache[BB] = BlockMask;
8248 }
8249
8250 // This is the block mask. We OR all incoming edges.
8251 for (auto *Predecessor : predecessors(BB)) {
8252 VPValue *EdgeMask = createEdgeMask(Predecessor, BB, Plan);
8253 if (!EdgeMask) // Mask of predecessor is all-one so mask of block is too.
8254 return BlockMaskCache[BB] = EdgeMask;
8255
8256 if (!BlockMask) { // BlockMask has its initialized nullptr value.
8257 BlockMask = EdgeMask;
8258 continue;
8259 }
8260
8261 BlockMask = Builder.createOr(BlockMask, EdgeMask);
8262 }
8263
8264 return BlockMaskCache[BB] = BlockMask;
8265}
8266
8267VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I, VFRange &Range,
8268 VPlanPtr &Plan) {
8269 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
8270 "Must be called with either a load or store");
8271
8272 auto willWiden = [&](ElementCount VF) -> bool {
8273 if (VF.isScalar())
8274 return false;
8275 LoopVectorizationCostModel::InstWidening Decision =
8276 CM.getWideningDecision(I, VF);
8277 assert(Decision != LoopVectorizationCostModel::CM_Unknown &&
8278 "CM decision should be taken at this point.");
8279 if (Decision == LoopVectorizationCostModel::CM_Interleave)
8280 return true;
8281 if (CM.isScalarAfterVectorization(I, VF) ||
8282 CM.isProfitableToScalarize(I, VF))
8283 return false;
8284 return Decision != LoopVectorizationCostModel::CM_Scalarize;
8285 };
8286
8287 if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
8288 return nullptr;
8289
8290 VPValue *Mask = nullptr;
8291 if (Legal->isMaskRequired(I))
8292 Mask = createBlockInMask(I->getParent(), Plan);
8293
8294 VPValue *Addr = Plan->getOrAddVPValue(getLoadStorePointerOperand(I));
8295 if (LoadInst *Load = dyn_cast<LoadInst>(I))
8296 return new VPWidenMemoryInstructionRecipe(*Load, Addr, Mask);
8297
8298 StoreInst *Store = cast<StoreInst>(I);
8299 VPValue *StoredValue = Plan->getOrAddVPValue(Store->getValueOperand());
8300 return new VPWidenMemoryInstructionRecipe(*Store, Addr, StoredValue, Mask);
8301}
8302
8303VPWidenIntOrFpInductionRecipe *
8304VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi, VPlan &Plan) const {
8305 // Check if this is an integer or fp induction. If so, build the recipe that
8306 // produces its scalar and vector values.
8307 InductionDescriptor II = Legal->getInductionVars().lookup(Phi);
8308 if (II.getKind() == InductionDescriptor::IK_IntInduction ||
8309 II.getKind() == InductionDescriptor::IK_FpInduction) {
8310 VPValue *Start = Plan.getOrAddVPValue(II.getStartValue());
8311 return new VPWidenIntOrFpInductionRecipe(Phi, Start);
8312 }
8313
8314 return nullptr;
8315}
8316
8317VPWidenIntOrFpInductionRecipe *
8318VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range,
8319 VPlan &Plan) const {
8320 // Optimize the special case where the source is a constant integer
8321 // induction variable. Notice that we can only optimize the 'trunc' case
8322 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and
8323 // (c) other casts depend on pointer size.
8324
8325 // Determine whether \p K is a truncation based on an induction variable that
8326 // can be optimized.
8327 auto isOptimizableIVTruncate =
8328 [&](Instruction *K) -> std::function<bool(ElementCount)> {
8329 return [=](ElementCount VF) -> bool {
8330 return CM.isOptimizableIVTruncate(K, VF);
8331 };
8332 };
8333
8334 if (LoopVectorizationPlanner::getDecisionAndClampRange(
8335 isOptimizableIVTruncate(I), Range)) {
8336
8337 InductionDescriptor II =
8338 Legal->getInductionVars().lookup(cast<PHINode>(I->getOperand(0)));
8339 VPValue *Start = Plan.getOrAddVPValue(II.getStartValue());
8340 return new VPWidenIntOrFpInductionRecipe(cast<PHINode>(I->getOperand(0)),
8341 Start, I);
8342 }
8343 return nullptr;
8344}
8345
8346VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi, VPlanPtr &Plan) {
8347 // We know that all PHIs in non-header blocks are converted into selects, so
8348 // we don't have to worry about the insertion order and we can just use the
8349 // builder. At this point we generate the predication tree. There may be
8350 // duplications since this is a simple recursive scan, but future
8351 // optimizations will clean it up.
8352
8353 SmallVector<VPValue *, 2> Operands;
8354 unsigned NumIncoming = Phi->getNumIncomingValues();
8355 for (unsigned In = 0; In < NumIncoming; In++) {
8356 VPValue *EdgeMask =
8357 createEdgeMask(Phi->getIncomingBlock(In), Phi->getParent(), Plan);
8358 assert((EdgeMask || NumIncoming == 1) &&
8359 "Multiple predecessors with one having a full mask");
8360 Operands.push_back(Plan->getOrAddVPValue(Phi->getIncomingValue(In)));
8361 if (EdgeMask)
8362 Operands.push_back(EdgeMask);
8363 }
8364 return new VPBlendRecipe(Phi, Operands);
8365}
8366
8367VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, VFRange &Range,
8368 VPlan &Plan) const {
8369
8370 bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
8371 [this, CI](ElementCount VF) {
8372 return CM.isScalarWithPredication(CI, VF);
8373 },
8374 Range);
8375
8376 if (IsPredicated)
8377 return nullptr;
8378
8379 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
8380 if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end ||
8381 ID == Intrinsic::lifetime_start || ID == Intrinsic::sideeffect ||
8382 ID == Intrinsic::pseudoprobe ||
8383 ID == Intrinsic::experimental_noalias_scope_decl))
8384 return nullptr;
8385
8386 auto willWiden = [&](ElementCount VF) -> bool {
8387 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
8388 // The following case may be scalarized depending on the VF.
8389 // The flag shows whether we use Intrinsic or a usual Call for vectorized
8390 // version of the instruction.
8391 // Is it beneficial to perform intrinsic call compared to lib call?
8392 bool NeedToScalarize = false;
8393 InstructionCost CallCost = CM.getVectorCallCost(CI, VF, NeedToScalarize);
8394 InstructionCost IntrinsicCost = ID ? CM.getVectorIntrinsicCost(CI, VF) : 0;
8395 bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost;
8396 assert(IntrinsicCost.isValid() && CallCost.isValid() &&
8397 "Cannot have invalid costs while widening");
8398 return UseVectorIntrinsic || !NeedToScalarize;
8399 };
8400
8401 if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
8402 return nullptr;
8403
8404 return new VPWidenCallRecipe(*CI, Plan.mapToVPValues(CI->arg_operands()));
8405}
8406
8407bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
8408 assert(!isa<BranchInst>(I) && !isa<PHINode>(I) && !isa<LoadInst>(I) &&
8409 !isa<StoreInst>(I) && "Instruction should have been handled earlier");
8410 // Instruction should be widened, unless it is scalar after vectorization,
8411 // scalarization is profitable or it is predicated.
8412 auto WillScalarize = [this, I](ElementCount VF) -> bool {
8413 return CM.isScalarAfterVectorization(I, VF) ||
8414 CM.isProfitableToScalarize(I, VF) ||
8415 CM.isScalarWithPredication(I, VF);
8416 };
8417 return !LoopVectorizationPlanner::getDecisionAndClampRange(WillScalarize,
8418 Range);
8419}
8420
8421VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VPlan &Plan) const {
8422 auto IsVectorizableOpcode = [](unsigned Opcode) {
8423 switch (Opcode) {
8424 case Instruction::Add:
8425 case Instruction::And:
8426 case Instruction::AShr:
8427 case Instruction::BitCast:
8428 case Instruction::FAdd:
8429 case Instruction::FCmp:
8430 case Instruction::FDiv:
8431 case Instruction::FMul:
8432 case Instruction::FNeg:
8433 case Instruction::FPExt:
8434 case Instruction::FPToSI:
8435 case Instruction::FPToUI:
8436 case Instruction::FPTrunc:
8437 case Instruction::FRem:
8438 case Instruction::FSub:
8439 case Instruction::ICmp:
8440 case Instruction::IntToPtr:
8441 case Instruction::LShr:
8442 case Instruction::Mul:
8443 case Instruction::Or:
8444 case Instruction::PtrToInt:
8445 case Instruction::SDiv:
8446 case Instruction::Select:
8447 case Instruction::SExt:
8448 case Instruction::Shl:
8449 case Instruction::SIToFP:
8450 case Instruction::SRem:
8451 case Instruction::Sub:
8452 case Instruction::Trunc:
8453 case Instruction::UDiv:
8454 case Instruction::UIToFP:
8455 case Instruction::URem:
8456 case Instruction::Xor:
8457 case Instruction::ZExt:
8458 return true;
8459 }
8460 return false;
8461 };
8462
8463 if (!IsVectorizableOpcode(I->getOpcode()))
8464 return nullptr;
8465
8466 // Success: widen this instruction.
8467 return new VPWidenRecipe(*I, Plan.mapToVPValues(I->operands()));
8468}
8469
8470VPBasicBlock *VPRecipeBuilder::handleReplication(
8471 Instruction *I, VFRange &Range, VPBasicBlock *VPBB,
8472 DenseMap<Instruction *, VPReplicateRecipe *> &PredInst2Recipe,
8473 VPlanPtr &Plan) {
8474 bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
8475 [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
8476 Range);
8477
8478 bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
8479 [&](ElementCount VF) { return CM.isScalarWithPredication(I, VF); },
8480 Range);
8481
8482 auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()),
8483 IsUniform, IsPredicated);
8484 setRecipe(I, Recipe);
8485 Plan->addVPValue(I, Recipe);
8486
8487 // Find if I uses a predicated instruction. If so, it will use its scalar
8488 // value. Avoid hoisting the insert-element which packs the scalar value into
8489 // a vector value, as that happens iff all users use the vector value.
8490 for (auto &Op : I->operands())
8491 if (auto *PredInst = dyn_cast<Instruction>(Op))
8492 if (PredInst2Recipe.find(PredInst) != PredInst2Recipe.end())
8493 PredInst2Recipe[PredInst]->setAlsoPack(false);
8494
8495 // Finalize the recipe for Instr, first if it is not predicated.
8496 if (!IsPredicated) {
8497 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
8498 VPBB->appendRecipe(Recipe);
8499 return VPBB;
8500 }
8501 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
8502 assert(VPBB->getSuccessors().empty() &&
8503 "VPBB has successors when handling predicated replication.");
8504 // Record predicated instructions for above packing optimizations.
8505 PredInst2Recipe[I] = Recipe;
8506 VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan);
8507 VPBlockUtils::insertBlockAfter(Region, VPBB);
8508 auto *RegSucc = new VPBasicBlock();
8509 VPBlockUtils::insertBlockAfter(RegSucc, Region);
8510 return RegSucc;
8511}
8512
8513VPRegionBlock *VPRecipeBuilder::createReplicateRegion(Instruction *Instr,
8514 VPRecipeBase *PredRecipe,
8515 VPlanPtr &Plan) {
8516 // Instructions marked for predication are replicated and placed under an
8517 // if-then construct to prevent side-effects.
8518
8519 // Generate recipes to compute the block mask for this region.
8520 VPValue *BlockInMask = createBlockInMask(Instr->getParent(), Plan);
8521
8522 // Build the triangular if-then region.
8523 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
8524 assert(Instr->getParent() && "Predicated instruction not in any basic block");
8525 auto *BOMRecipe = new VPBranchOnMaskRecipe(BlockInMask);
8526 auto *Entry = new VPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe);
8527 auto *PHIRecipe = Instr->getType()->isVoidTy()
8528 ? nullptr
8529 : new VPPredInstPHIRecipe(Plan->getOrAddVPValue(Instr));
8530 auto *Exit = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
8531 auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", PredRecipe);
8532 VPRegionBlock *Region = new VPRegionBlock(Entry, Exit, RegionName, true);
8533
8534 // Note: first set Entry as region entry and then connect successors starting
8535 // from it in order, to propagate the "parent" of each VPBasicBlock.
8536 VPBlockUtils::insertTwoBlocksAfter(Pred, Exit, BlockInMask, Entry);
8537 VPBlockUtils::connectBlocks(Pred, Exit);
8538
8539 return Region;
8540}
8541
8542VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
8543 VFRange &Range,
8544 VPlanPtr &Plan) {
8545 // First, check for specific widening recipes that deal with calls, memory
8546 // operations, inductions and Phi nodes.
8547 if (auto *CI = dyn_cast<CallInst>(Instr))
8548 return tryToWidenCall(CI, Range, *Plan);
8549
8550 if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
8551 return tryToWidenMemory(Instr, Range, Plan);
8552
8553 VPRecipeBase *Recipe;
8554 if (auto Phi = dyn_cast<PHINode>(Instr)) {
8555 if (Phi->getParent() != OrigLoop->getHeader())
8556 return tryToBlend(Phi, Plan);
8557 if ((Recipe = tryToOptimizeInductionPHI(Phi, *Plan)))
8558 return Recipe;
8559
8560 if (Legal->isReductionVariable(Phi)) {
8561 RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
8562 VPValue *StartV =
8563 Plan->getOrAddVPValue(RdxDesc.getRecurrenceStartValue());
8564 return new VPWidenPHIRecipe(Phi, RdxDesc, *StartV);
8565 }
8566
8567 return new VPWidenPHIRecipe(Phi);
8568 }
8569
8570 if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8571 cast<TruncInst>(Instr), Range, *Plan)))
8572 return Recipe;
8573
8574 if (!shouldWiden(Instr, Range))
8575 return nullptr;
8576
8577 if (auto GEP = dyn_cast<GetElementPtrInst>(Instr))
8578 return new VPWidenGEPRecipe(GEP, Plan->mapToVPValues(GEP->operands()),
8579 OrigLoop);
8580
8581 if (auto *SI = dyn_cast<SelectInst>(Instr)) {
8582 bool InvariantCond =
8583 PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop);
8584 return new VPWidenSelectRecipe(*SI, Plan->mapToVPValues(SI->operands()),
8585 InvariantCond);
8586 }
8587
8588 return tryToWiden(Instr, *Plan);
8589}
8590
8591void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
8592 ElementCount MaxVF) {
8593 assert(OrigLoop->isInnermost() && "Inner loop expected.");
8594
8595 // Collect instructions from the original loop that will become trivially dead
8596 // in the vectorized loop. We don't need to vectorize these instructions. For
8597 // example, original induction update instructions can become dead because we
8598 // separately emit induction "steps" when generating code for the new loop.
8599 // Similarly, we create a new latch condition when setting up the structure
8600 // of the new loop, so the old one can become dead.
8601 SmallPtrSet<Instruction *, 4> DeadInstructions;
8602 collectTriviallyDeadInstructions(DeadInstructions);
8603
8604 // Add assume instructions we need to drop to DeadInstructions, to prevent
8605 // them from being added to the VPlan.
8606 // TODO: We only need to drop assumes in blocks that get flattend. If the
8607 // control flow is preserved, we should keep them.
8608 auto &ConditionalAssumes = Legal->getConditionalAssumes();
8609 DeadInstructions.insert(ConditionalAssumes.begin(), ConditionalAssumes.end());
8610
8611 DenseMap<Instruction *, Instruction *> &SinkAfter = Legal->getSinkAfter();
8612 // Dead instructions do not need sinking. Remove them from SinkAfter.
8613 for (Instruction *I : DeadInstructions)
8614 SinkAfter.erase(I);
8615
8616 auto MaxVFPlusOne = MaxVF.getWithIncrement(1);
8617 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
8618 VFRange SubRange = {VF, MaxVFPlusOne};
8619 VPlans.push_back(
8620 buildVPlanWithVPRecipes(SubRange, DeadInstructions, SinkAfter));
8621 VF = SubRange.End;
8622 }
8623}
8624
8625VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
8626 VFRange &Range, SmallPtrSetImpl<Instruction *> &DeadInstructions,
8627 const DenseMap<Instruction *, Instruction *> &SinkAfter) {
8628
8629 // Hold a mapping from predicated instructions to their recipes, in order to
8630 // fix their AlsoPack behavior if a user is determined to replicate and use a
8631 // scalar instead of vector value.
8632 DenseMap<Instruction *, VPReplicateRecipe *> PredInst2Recipe;
8633
8634 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
8635
8636 VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder);
8637
8638 // ---------------------------------------------------------------------------
8639 // Pre-construction: record ingredients whose recipes we'll need to further
8640 // process after constructing the initial VPlan.
8641 // ---------------------------------------------------------------------------
8642
8643 // Mark instructions we'll need to sink later and their targets as
8644 // ingredients whose recipe we'll need to record.
8645 for (auto &Entry : SinkAfter) {
8646 RecipeBuilder.recordRecipeOf(Entry.first);
8647 RecipeBuilder.recordRecipeOf(Entry.second);
8648 }
8649 for (auto &Reduction : CM.getInLoopReductionChains()) {
8650 PHINode *Phi = Reduction.first;
8651 RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind();
8652 const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;
8653
8654 RecipeBuilder.recordRecipeOf(Phi);
8655 for (auto &R : ReductionOperations) {
8656 RecipeBuilder.recordRecipeOf(R);
8657 // For min/max reducitons, where we have a pair of icmp/select, we also
8658 // need to record the ICmp recipe, so it can be removed later.
8659 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
8660 RecipeBuilder.recordRecipeOf(cast<Instruction>(R->getOperand(0)));
8661 }
8662 }
8663
8664 // For each interleave group which is relevant for this (possibly trimmed)
8665 // Range, add it to the set of groups to be later applied to the VPlan and add
8666 // placeholders for its members' Recipes which we'll be replacing with a
8667 // single VPInterleaveRecipe.
8668 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
8669 auto applyIG = [IG, this](ElementCount VF) -> bool {
8670 return (VF.isVector() && // Query is illegal for VF == 1
8671 CM.getWideningDecision(IG->getInsertPos(), VF) ==
8672 LoopVectorizationCostModel::CM_Interleave);
8673 };
8674 if (!getDecisionAndClampRange(applyIG, Range))
8675 continue;
8676 InterleaveGroups.insert(IG);
8677 for (unsigned i = 0; i < IG->getFactor(); i++)
8678 if (Instruction *Member = IG->getMember(i))
8679 RecipeBuilder.recordRecipeOf(Member);
8680 };
8681
8682 // ---------------------------------------------------------------------------
8683 // Build initial VPlan: Scan the body of the loop in a topological order to
8684 // visit each basic block after having visited its predecessor basic blocks.
8685 // ---------------------------------------------------------------------------
8686
8687 // Create a dummy pre-entry VPBasicBlock to start building the VPlan.
8688 auto Plan = std::make_unique<VPlan>();
8689 VPBasicBlock *VPBB = new VPBasicBlock("Pre-Entry");
8690 Plan->setEntry(VPBB);
8691
8692 // Scan the body of the loop in a topological order to visit each basic block
8693 // after having visited its predecessor basic blocks.
8694 LoopBlocksDFS DFS(OrigLoop);
8695 DFS.perform(LI);
8696
8697 for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
8698 // Relevant instructions from basic block BB will be grouped into VPRecipe
8699 // ingredients and fill a new VPBasicBlock.
8700 unsigned VPBBsForBB = 0;
8701 auto *FirstVPBBForBB = new VPBasicBlock(BB->getName());
8702 VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB);
8703 VPBB = FirstVPBBForBB;
8704 Builder.setInsertPoint(VPBB);
8705
8706 // Introduce each ingredient into VPlan.
8707 // TODO: Model and preserve debug instrinsics in VPlan.
8708 for (Instruction &I : BB->instructionsWithoutDebug()) {
8709 Instruction *Instr = &I;
8710
8711 // First filter out irrelevant instructions, to ensure no recipes are
8712 // built for them.
8713 if (isa<BranchInst>(Instr) || DeadInstructions.count(Instr))
8714 continue;
8715
8716 if (auto Recipe =
8717 RecipeBuilder.tryToCreateWidenRecipe(Instr, Range, Plan)) {
8718 for (auto *Def : Recipe->definedValues()) {
8719 auto *UV = Def->getUnderlyingValue();
8720 Plan->addVPValue(UV, Def);
8721 }
8722
8723 RecipeBuilder.setRecipe(Instr, Recipe);
8724 VPBB->appendRecipe(Recipe);
8725 continue;
8726 }
8727
8728 // Otherwise, if all widening options failed, Instruction is to be
8729 // replicated. This may create a successor for VPBB.
8730 VPBasicBlock *NextVPBB = RecipeBuilder.handleReplication(
8731 Instr, Range, VPBB, PredInst2Recipe, Plan);
8732 if (NextVPBB != VPBB) {
8733 VPBB = NextVPBB;
8734 VPBB->setName(BB->hasName() ? BB->getName() + "." + Twine(VPBBsForBB++)
8735 : "");
8736 }
8737 }
8738 }
8739
8740 // Discard empty dummy pre-entry VPBasicBlock. Note that other VPBasicBlocks
8741 // may also be empty, such as the last one VPBB, reflecting original
8742 // basic-blocks with no recipes.
8743 VPBasicBlock *PreEntry = cast<VPBasicBlock>(Plan->getEntry());
8744 assert(PreEntry->empty() && "Expecting empty pre-entry block.");
8745 VPBlockBase *Entry = Plan->setEntry(PreEntry->getSingleSuccessor());
8746 VPBlockUtils::disconnectBlocks(PreEntry, Entry);
8747 delete PreEntry;
8748
8749 // ---------------------------------------------------------------------------
8750 // Transform initial VPlan: Apply previously taken decisions, in order, to
8751 // bring the VPlan to its final state.
8752 // ---------------------------------------------------------------------------
8753
8754 // Apply Sink-After legal constraints.
8755 for (auto &Entry : SinkAfter) {
8756 VPRecipeBase *Sink = RecipeBuilder.getRecipe(Entry.first);
8757 VPRecipeBase *Target = RecipeBuilder.getRecipe(Entry.second);
8758 // If the target is in a replication region, make sure to move Sink to the
8759 // block after it, not into the replication region itself.
8760 if (auto *Region =
8761 dyn_cast_or_null<VPRegionBlock>(Target->getParent()->getParent())) {
8762 if (Region->isReplicator()) {
8763 assert(Region->getNumSuccessors() == 1 && "Expected SESE region!");
8764 VPBasicBlock *NextBlock =
8765 cast<VPBasicBlock>(Region->getSuccessors().front());
8766 Sink->moveBefore(*NextBlock, NextBlock->getFirstNonPhi());
8767 continue;
8768 }
8769 }
8770 Sink->moveAfter(Target);
8771 }
8772
8773 // Interleave memory: for each Interleave Group we marked earlier as relevant
8774 // for this VPlan, replace the Recipes widening its memory instructions with a
8775 // single VPInterleaveRecipe at its insertion point.
8776 for (auto IG : InterleaveGroups) {
8777 auto *Recipe = cast<VPWidenMemoryInstructionRecipe>(
8778 RecipeBuilder.getRecipe(IG->getInsertPos()));
8779 SmallVector<VPValue *, 4> StoredValues;
8780 for (unsigned i = 0; i < IG->getFactor(); ++i)
8781 if (auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i)))
8782 StoredValues.push_back(Plan->getOrAddVPValue(SI->getOperand(0)));
8783
8784 auto *VPIG = new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues,
8785 Recipe->getMask());
8786 VPIG->insertBefore(Recipe);
8787 unsigned J = 0;
8788 for (unsigned i = 0; i < IG->getFactor(); ++i)
8789 if (Instruction *Member = IG->getMember(i)) {
8790 if (!Member->getType()->isVoidTy()) {
8791 VPValue *OriginalV = Plan->getVPValue(Member);
8792 Plan->removeVPValueFor(Member);
8793 Plan->addVPValue(Member, VPIG->getVPValue(J));
8794 OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
8795 J++;
8796 }
8797 RecipeBuilder.getRecipe(Member)->eraseFromParent();
8798 }
8799 }
8800
8801 // Adjust the recipes for any inloop reductions.
8802 if (Range.Start.isVector())
8803 adjustRecipesForInLoopReductions(Plan, RecipeBuilder);
8804
8805 // Finally, if tail is folded by masking, introduce selects between the phi
8806 // and the live-out instruction of each reduction, at the end of the latch.
8807 if (CM.foldTailByMasking() && !Legal->getReductionVars().empty()) {
8808 Builder.setInsertPoint(VPBB);
8809 auto *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan);
8810 for (auto &Reduction : Legal->getReductionVars()) {
8811 if (CM.isInLoopReduction(Reduction.first))
8812 continue;
8813 VPValue *Phi = Plan->getOrAddVPValue(Reduction.first);
8814 VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr());
8815 Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi});
8816 }
8817 }
8818
8819 std::string PlanName;
8820 raw_string_ostream RSO(PlanName);
8821 ElementCount VF = Range.Start;
8822 Plan->addVF(VF);
8823 RSO << "Initial VPlan for VF={" << VF;
8824 for (VF *= 2; ElementCount::isKnownLT(VF, Range.End); VF *= 2) {
8825 Plan->addVF(VF);
8826 RSO << "," << VF;
8827 }
8828 RSO << "},UF>=1";
8829 RSO.flush();
8830 Plan->setName(PlanName);
8831
8832 return Plan;
8833}
8834
8835VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
8836 // Outer loop handling: They may require CFG and instruction level
8837 // transformations before even evaluating whether vectorization is profitable.
8838 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
8839 // the vectorization pipeline.
8840 assert(!OrigLoop->isInnermost());
8841 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
8842
8843 // Create new empty VPlan
8844 auto Plan = std::make_unique<VPlan>();
8845
8846 // Build hierarchical CFG
8847 VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
8848 HCFGBuilder.buildHierarchicalCFG();
8849
8850 for (ElementCount VF = Range.Start; ElementCount::isKnownLT(VF, Range.End);
8851 VF *= 2)
8852 Plan->addVF(VF);
8853
8854 if (EnableVPlanPredication) {
8855 VPlanPredicator VPP(*Plan);
8856 VPP.predicate();
8857
8858 // Avoid running transformation to recipes until masked code generation in
8859 // VPlan-native path is in place.
8860 return Plan;
8861 }
8862
8863 SmallPtrSet<Instruction *, 1> DeadInstructions;
8864 VPlanTransforms::VPInstructionsToVPRecipes(
8865 OrigLoop, Plan, Legal->getInductionVars(), DeadInstructions);
8866 return Plan;
8867}
8868
8869// Adjust the recipes for any inloop reductions. The chain of instructions
8870// leading from the loop exit instr to the phi need to be converted to
8871// reductions, with one operand being vector and the other being the scalar
8872// reduction chain.
8873void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
8874 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder) {
8875 for (auto &Reduction : CM.getInLoopReductionChains()) {
8876 PHINode *Phi = Reduction.first;
8877 RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
8878 const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;
8879
8880 // ReductionOperations are orders top-down from the phi's use to the
8881 // LoopExitValue. We keep a track of the previous item (the Chain) to tell
8882 // which of the two operands will remain scalar and which will be reduced.
8883 // For minmax the chain will be the select instructions.
8884 Instruction *Chain = Phi;
8885 for (Instruction *R : ReductionOperations) {
8886 VPRecipeBase *WidenRecipe = RecipeBuilder.getRecipe(R);
8887 RecurKind Kind = RdxDesc.getRecurrenceKind();
8888
8889 VPValue *ChainOp = Plan->getVPValue(Chain);
8890 unsigned FirstOpId;
8891 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
8892 assert(isa<VPWidenSelectRecipe>(WidenRecipe) &&
8893 "Expected to replace a VPWidenSelectSC");
8894 FirstOpId = 1;
8895 } else {
8896 assert(isa<VPWidenRecipe>(WidenRecipe) &&
8897 "Expected to replace a VPWidenSC");
8898 FirstOpId = 0;
8899 }
8900 unsigned VecOpId =
8901 R->getOperand(FirstOpId) == Chain ? FirstOpId + 1 : FirstOpId;
8902 VPValue *VecOp = Plan->getVPValue(R->getOperand(VecOpId));
8903
8904 auto *CondOp = CM.foldTailByMasking()
8905 ? RecipeBuilder.createBlockInMask(R->getParent(), Plan)
8906 : nullptr;
8907 VPReductionRecipe *RedRecipe = new VPReductionRecipe(
8908 &RdxDesc, R, ChainOp, VecOp, CondOp, Legal->hasFunNoNaNAttr(), TTI);
8909 WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe);
8910 Plan->removeVPValueFor(R);
8911 Plan->addVPValue(R, RedRecipe);
8912 WidenRecipe->getParent()->insert(RedRecipe, WidenRecipe->getIterator());
8913 WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe);
8914 WidenRecipe->eraseFromParent();
8915
8916 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
8917 VPRecipeBase *CompareRecipe =
8918 RecipeBuilder.getRecipe(cast<Instruction>(R->getOperand(0)));
8919 assert(isa<VPWidenRecipe>(CompareRecipe) &&
8920 "Expected to replace a VPWidenSC");
8921 assert(cast<VPWidenRecipe>(CompareRecipe)->getNumUsers() == 0 &&
8922 "Expected no remaining users");
8923 CompareRecipe->eraseFromParent();
8924 }
8925 Chain = R;
8926 }
8927 }
8928}
8929
8930Value* LoopVectorizationPlanner::VPCallbackILV::
8931getOrCreateVectorValues(Value *V, unsigned Part) {
8932 return ILV.getOrCreateVectorValue(V, Part);
8933}
8934
8935Value *LoopVectorizationPlanner::VPCallbackILV::getOrCreateScalarValue(
8936 Value *V, const VPIteration &Instance) {
8937 return ILV.getOrCreateScalarValue(V, Instance);
8938}
8939
8940void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
8941 VPSlotTracker &SlotTracker) const {
8942 O << "\"INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";
8943 IG->getInsertPos()->printAsOperand(O, false);
8944 O << ", ";
8945 getAddr()->printAsOperand(O, SlotTracker);
8946 VPValue *Mask = getMask();
8947 if (Mask) {
8948 O << ", ";
8949 Mask->printAsOperand(O, SlotTracker);
8950 }
8951 for (unsigned i = 0; i < IG->getFactor(); ++i)
8952 if (Instruction *I = IG->getMember(i))
8953 O << "\\l\" +\n" << Indent << "\" " << VPlanIngredient(I) << " " << i;
8954}
8955
8956void VPWidenCallRecipe::execute(VPTransformState &State) {
8957 State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this,
8958 *this, State);
8959}
8960
8961void VPWidenSelectRecipe::execute(VPTransformState &State) {
8962 State.ILV->widenSelectInstruction(*cast<SelectInst>(getUnderlyingInstr()),
8963 this, *this, InvariantCond, State);
8964}
8965
8966void VPWidenRecipe::execute(VPTransformState &State) {
8967 State.ILV->widenInstruction(*getUnderlyingInstr(), this, *this, State);
8968}
8969
8970void VPWidenGEPRecipe::execute(VPTransformState &State) {
8971 State.ILV->widenGEP(cast<GetElementPtrInst>(getUnderlyingInstr()), this,
8972 *this, State.UF, State.VF, IsPtrLoopInvariant,
8973 IsIndexLoopInvariant, State);
8974}
8975
8976void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
8977 assert(!State.Instance && "Int or FP induction being replicated.");
8978 State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(),
8979 Trunc);
8980}
8981
8982void VPWidenPHIRecipe::execute(VPTransformState &State) {
8983 Value *StartV =
8984 getStartValue() ? getStartValue()->getLiveInIRValue() : nullptr;
8985 State.ILV->widenPHIInstruction(Phi, RdxDesc, StartV, State.UF, State.VF);
8986}
8987
8988void VPBlendRecipe::execute(VPTransformState &State) {
8989 State.ILV->setDebugLocFromInst(State.Builder, Phi);
8990 // We know that all PHIs in non-header blocks are converted into
8991 // selects, so we don't have to worry about the insertion order and we
8992 // can just use the builder.
8993 // At this point we generate the predication tree. There may be
8994 // duplications since this is a simple recursive scan, but future
8995 // optimizations will clean it up.
8996
8997 unsigned NumIncoming = getNumIncomingValues();
8998
8999 // Generate a sequence of selects of the form:
9000 // SELECT(Mask3, In3,
9001 // SELECT(Mask2, In2,
9002 // SELECT(Mask1, In1,
9003 // In0)))
9004 // Note that Mask0 is never used: lanes for which no path reaches this phi and
9005 // are essentially undef are taken from In0.
9006 InnerLoopVectorizer::VectorParts Entry(State.UF);
9007 for (unsigned In = 0; In < NumIncoming; ++In) {
9008 for (unsigned Part = 0; Part < State.UF; ++Part) {
9009 // We might have single edge PHIs (blocks) - use an identity
9010 // 'select' for the first PHI operand.
9011 Value *In0 = State.get(getIncomingValue(In), Part);
9012 if (In == 0)
9013 Entry[Part] = In0; // Initialize with the first incoming value.
9014 else {
9015 // Select between the current value and the previous incoming edge
9016 // based on the incoming mask.
9017 Value *Cond = State.get(getMask(In), Part);
9018 Entry[Part] =
9019 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
9020 }
9021 }
9022 }
9023 for (unsigned Part = 0; Part < State.UF; ++Part)
9024 State.ValueMap.setVectorValue(Phi, Part, Entry[Part]);
9025}
9026
9027void VPInterleaveRecipe::execute(VPTransformState &State) {
9028 assert(!State.Instance && "Interleave group being replicated.");
9029 State.ILV->vectorizeInterleaveGroup(IG, definedValues(), State, getAddr(),
9030 getStoredValues(), getMask());
9031}
9032
9033void VPReductionRecipe::execute(VPTransformState &State) {
9034 assert(!State.Instance && "Reduction being replicated.");
9035 for (unsigned Part = 0; Part < State.UF; ++Part) {
9036 RecurKind Kind = RdxDesc->getRecurrenceKind();
9037 Value *NewVecOp = State.get(getVecOp(), Part);
9038 if (VPValue *Cond = getCondOp()) {
9039 Value *NewCond = State.get(Cond, Part);
9040 VectorType *VecTy = cast<VectorType>(NewVecOp->getType());
9041 Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
9042 Kind, VecTy->getElementType());
9043 Constant *IdenVec =
9044 ConstantVector::getSplat(VecTy->getElementCount(), Iden);
9045 Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, IdenVec);
9046 NewVecOp = Select;
9047 }
9048 Value *NewRed =
9049 createTargetReduction(State.Builder, TTI, *RdxDesc, NewVecOp);
9050 Value *PrevInChain = State.get(getChainOp(), Part);
9051 Value *NextInChain;
9052 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
9053 NextInChain =
9054 createMinMaxOp(State.Builder, RdxDesc->getRecurrenceKind(),
9055 NewRed, PrevInChain);
9056 } else {
9057 NextInChain = State.Builder.CreateBinOp(
9058 (Instruction::BinaryOps)getUnderlyingInstr()->getOpcode(), NewRed,
9059 PrevInChain);
9060 }
9061 State.set(this, getUnderlyingInstr(), NextInChain, Part);
9062 }
9063}
9064
9065void VPReplicateRecipe::execute(VPTransformState &State) {
9066 if (State.Instance) { // Generate a single instance.
9067 assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
9068 State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this,
9069 *State.Instance, IsPredicated, State);
9070 // Insert scalar instance packing it into a vector.
9071 if (AlsoPack && State.VF.isVector()) {
9072 // If we're constructing lane 0, initialize to start from poison.
9073 if (State.Instance->Lane == 0) {
9074 assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
9075 Value *Poison = PoisonValue::get(
9076 VectorType::get(getUnderlyingValue()->getType(), State.VF));
9077 State.ValueMap.setVectorValue(getUnderlyingInstr(),
9078 State.Instance->Part, Poison);
9079 }
9080 State.ILV->packScalarIntoVectorValue(getUnderlyingInstr(),
9081 *State.Instance);
9082 }
9083 return;
9084 }
9085
9086 // Generate scalar instances for all VF lanes of all UF parts, unless the
9087 // instruction is uniform inwhich case generate only the first lane for each
9088 // of the UF parts.
9089 unsigned EndLane = IsUniform ? 1 : State.VF.getKnownMinValue();
9090 assert((!State.VF.isScalable() || IsUniform) &&
9091 "Can't scalarize a scalable vector");
9092 for (unsigned Part = 0; Part < State.UF; ++Part)
9093 for (unsigned Lane = 0; Lane < EndLane; ++Lane)
9094 State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this, {Part, Lane},
9095 IsPredicated, State);
9096}
9097
9098void VPBranchOnMaskRecipe::execute(VPTransformState &State) {
9099 assert(State.Instance && "Branch on Mask works only on single instance.");
9100
9101 unsigned Part = State.Instance->Part;
9102 unsigned Lane = State.Instance->Lane;
9103
9104 Value *ConditionBit = nullptr;
9105 VPValue *BlockInMask = getMask();
9106 if (BlockInMask) {
9107 ConditionBit = State.get(BlockInMask, Part);
9108 if (ConditionBit->getType()->isVectorTy())
9109 ConditionBit = State.Builder.CreateExtractElement(
9110 ConditionBit, State.Builder.getInt32(Lane));
9111 } else // Block in mask is all-one.
9112 ConditionBit = State.Builder.getTrue();
9113
9114 // Replace the temporary unreachable terminator with a new conditional branch,
9115 // whose two destinations will be set later when they are created.
9116 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
9117 assert(isa<UnreachableInst>(CurrentTerminator) &&
9118 "Expected to replace unreachable terminator with conditional branch.");
9119 auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
9120 CondBr->setSuccessor(0, nullptr);
9121 ReplaceInstWithInst(CurrentTerminator, CondBr);
9122}
9123
9124void VPPredInstPHIRecipe::execute(VPTransformState &State) {
9125 assert(State.Instance && "Predicated instruction PHI works per instance.");
9126 Instruction *ScalarPredInst =
9127 cast<Instruction>(State.get(getOperand(0), *State.Instance));
9128 BasicBlock *PredicatedBB = ScalarPredInst->getParent();
9129 BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
9130 assert(PredicatingBB && "Predicated block has no single predecessor.");
9131
9132 // By current pack/unpack logic we need to generate only a single phi node: if
9133 // a vector value for the predicated instruction exists at this point it means
9134 // the instruction has vector users only, and a phi for the vector value is
9135 // needed. In this case the recipe of the predicated instruction is marked to
9136 // also do that packing, thereby "hoisting" the insert-element sequence.
9137 // Otherwise, a phi node for the scalar value is needed.
9138 unsigned Part = State.Instance->Part;
9139 Instruction *PredInst =
9140 cast<Instruction>(getOperand(0)->getUnderlyingValue());
9141 if (State.ValueMap.hasVectorValue(PredInst, Part)) {
9142 Value *VectorValue = State.ValueMap.getVectorValue(PredInst, Part);
9143 InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
9144 PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
9145 VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
9146 VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
9147 State.ValueMap.resetVectorValue(PredInst, Part, VPhi); // Update cache.
9148 } else {
9149 Type *PredInstType = PredInst->getType();
9150 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
9151 Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()), PredicatingBB);
9152 Phi->addIncoming(ScalarPredInst, PredicatedBB);
9153 State.ValueMap.resetScalarValue(PredInst, *State.Instance, Phi);
9154 }
9155}
9156
9157void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
9158 VPValue *StoredValue = isStore() ? getStoredValue() : nullptr;
9159 State.ILV->vectorizeMemoryInstruction(&Ingredient, State,
9160 StoredValue ? nullptr : getVPValue(),
9161 getAddr(), StoredValue, getMask());
9162}
9163
9164// Determine how to lower the scalar epilogue, which depends on 1) optimising
9165// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
9166// predication, and 4) a TTI hook that analyses whether the loop is suitable
9167// for predication.
9168static ScalarEpilogueLowering getScalarEpilogueLowering(
9169 Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI,
9170 BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
9171 AssumptionCache *AC, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
9172 LoopVectorizationLegality &LVL) {
9173 // 1) OptSize takes precedence over all other options, i.e. if this is set,
9174 // don't look at hints or options, and don't request a scalar epilogue.
9175 // (For PGSO, as shouldOptimizeForSize isn't currently accessible from
9176 // LoopAccessInfo (due to code dependency and not being able to reliably get
9177 // PSI/BFI from a loop analysis under NPM), we cannot suppress the collection
9178 // of strides in LoopAccessInfo::analyzeLoop() and vectorize without
9179 // versioning when the vectorization is forced, unlike hasOptSize. So revert
9180 // back to the old way and vectorize with versioning when forced. See D81345.)
9181 if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
9182 PGSOQueryType::IRPass) &&
9183 Hints.getForce() != LoopVectorizeHints::FK_Enabled))
9184 return CM_ScalarEpilogueNotAllowedOptSize;
9185
9186 // 2) If set, obey the directives
9187 if (PreferPredicateOverEpilogue.getNumOccurrences()) {
9188 switch (PreferPredicateOverEpilogue) {
9189 case PreferPredicateTy::ScalarEpilogue:
9190 return CM_ScalarEpilogueAllowed;
9191 case PreferPredicateTy::PredicateElseScalarEpilogue:
9192 return CM_ScalarEpilogueNotNeededUsePredicate;
9193 case PreferPredicateTy::PredicateOrDontVectorize:
9194 return CM_ScalarEpilogueNotAllowedUsePredicate;
9195 };
9196 }
9197
9198 // 3) If set, obey the hints
9199 switch (Hints.getPredicate()) {
9200 case LoopVectorizeHints::FK_Enabled:
9201 return CM_ScalarEpilogueNotNeededUsePredicate;
9202 case LoopVectorizeHints::FK_Disabled:
9203 return CM_ScalarEpilogueAllowed;
9204 };
9205
9206 // 4) if the TTI hook indicates this is profitable, request predication.
9207 if (TTI->preferPredicateOverEpilogue(L, LI, *SE, *AC, TLI, DT,
9208 LVL.getLAI()))
9209 return CM_ScalarEpilogueNotNeededUsePredicate;
9210
9211 return CM_ScalarEpilogueAllowed;
9212}
9213
9214void VPTransformState::set(VPValue *Def, Value *IRDef, Value *V,
9215 unsigned Part) {
9216 set(Def, V, Part);
9217 ILV->setVectorValue(IRDef, Part, V);
9218}
9219
9220// Process the loop in the VPlan-native vectorization path. This path builds
9221// VPlan upfront in the vectorization pipeline, which allows to apply
9222// VPlan-to-VPlan transformations from the very beginning without modifying the
9223// input LLVM IR.
9224static bool processLoopInVPlanNativePath(
9225 Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT,
9226 LoopVectorizationLegality *LVL, TargetTransformInfo *TTI,
9227 TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC,
9228 OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI,
9229 ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints) {
9230
9231 if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
9232 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
9233 return false;
9234 }
9235 assert(EnableVPlanNativePath && "VPlan-native path is disabled.");
9236 Function *F = L->getHeader()->getParent();
9237 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());
9238
9239 ScalarEpilogueLowering SEL = getScalarEpilogueLowering(
9240 F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, *LVL);
9241
9242 LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,
9243 &Hints, IAI);
9244 // Use the planner for outer loop vectorization.
9245 // TODO: CM is not used at this point inside the planner. Turn CM into an
9246 // optional argument if we don't need it in the future.
9247 LoopVectorizationPlanner LVP(L, LI, TLI, TTI, LVL, CM, IAI, PSE);
9248
9249 // Get user vectorization factor.
9250 ElementCount UserVF = Hints.getWidth();
9251
9252 // Plan how to best vectorize, return the best VF and its cost.
9253 const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF);
9254
9255 // If we are stress testing VPlan builds, do not attempt to generate vector
9256 // code. Masked vector code generation support will follow soon.
9257 // Also, do not attempt to vectorize if no vector code will be produced.
9258 if (VPlanBuildStressTest || EnableVPlanPredication ||
9259 VectorizationFactor::Disabled() == VF)
9260 return false;
9261
9262 LVP.setBestPlan(VF.Width, 1);
9263
9264 InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, 1, LVL,
9265 &CM, BFI, PSI);
9266 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
9267 << L->getHeader()->getParent()->getName() << "\"\n");
9268 LVP.executePlan(LB, DT);
9269
9270 // Mark the loop as already vectorized to avoid vectorizing again.
9271 Hints.setAlreadyVectorized();
9272
9273 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()));
9274 return true;
9275}
9276
9277LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
9278 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
9279 !EnableLoopInterleaving),
9280 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
9281 !EnableLoopVectorization) {}
9282
9283bool LoopVectorizePass::processLoop(Loop *L) {
9284 assert((EnableVPlanNativePath || L->isInnermost()) &&
9285 "VPlan-native path is not enabled. Only process inner loops.");
9286
9287#ifndef NDEBUG
9288 const std::string DebugLocStr = getDebugLocString(L);
9289#endif /* NDEBUG */
9290
9291 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in \""
9292 << L->getHeader()->getParent()->getName() << "\" from "
9293 << DebugLocStr << "\n");
9294
9295 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE);
9296
9297 LLVM_DEBUG(
9298 dbgs() << "LV: Loop hints:"
9299 << " force="
9300 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
9301 ? "disabled"
9302 : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
9303 ? "enabled"
9304 : "?"))
9305 << " width=" << Hints.getWidth()
9306 << " unroll=" << Hints.getInterleave() << "\n");
9307
9308 // Function containing loop
9309 Function *F = L->getHeader()->getParent();
9310
9311 // Looking at the diagnostic output is the only way to determine if a loop
9312 // was vectorized (other than looking at the IR or machine code), so it
9313 // is important to generate an optimization remark for each loop. Most of
9314 // these messages are generated as OptimizationRemarkAnalysis. Remarks
9315 // generated as OptimizationRemark and OptimizationRemarkMissed are
9316 // less verbose reporting vectorized loops and unvectorized loops that may
9317 // benefit from vectorization, respectively.
9318
9319 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
9320 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
9321 return false;
9322 }
9323
9324 PredicatedScalarEvolution PSE(*SE, *L);
9325
9326 // Check if it is legal to vectorize the loop.
9327 LoopVectorizationRequirements Requirements(*ORE);
9328 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, GetLAA, LI, ORE,
9329 &Requirements, &Hints, DB, AC, BFI, PSI);
9330 if (!LVL.canVectorize(EnableVPlanNativePath)) {
9331 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
9332 Hints.emitRemarkWithHints();
9333 return false;
9334 }
9335
9336 // Check the function attributes and profiles to find out if this function
9337 // should be optimized for size.
9338 ScalarEpilogueLowering SEL = getScalarEpilogueLowering(
9339 F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, LVL);
9340
9341 // Entrance to the VPlan-native vectorization path. Outer loops are processed
9342 // here. They may require CFG and instruction level transformations before
9343 // even evaluating whether vectorization is profitable. Since we cannot modify
9344 // the incoming IR, we need to build VPlan upfront in the vectorization
9345 // pipeline.
9346 if (!L->isInnermost())
9347 return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC,
9348 ORE, BFI, PSI, Hints);
9349
9350 assert(L->isInnermost() && "Inner loop expected.");
9351
9352 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
9353 // count by optimizing for size, to minimize overheads.
9354 auto ExpectedTC = getSmallBestKnownTC(*SE, L);
9355 if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) {
9356 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
9357 << "This loop is worth vectorizing only if no scalar "
9358 << "iteration overheads are incurred.");
9359 if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
9360 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
9361 else {
9362 LLVM_DEBUG(dbgs() << "\n");
9363 SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
9364 }
9365 }
9366
9367 // Check the function attributes to see if implicit floats are allowed.
9368 // FIXME: This check doesn't seem possibly correct -- what if the loop is
9369 // an integer loop and the vector instructions selected are purely integer
9370 // vector instructions?
9371 if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
9372 reportVectorizationFailure(
9373 "Can't vectorize when the NoImplicitFloat attribute is used",
9374 "loop not vectorized due to NoImplicitFloat attribute",
9375 "NoImplicitFloat", ORE, L);
9376 Hints.emitRemarkWithHints();
9377 return false;
9378 }
9379
9380 // Check if the target supports potentially unsafe FP vectorization.
9381 // FIXME: Add a check for the type of safety issue (denormal, signaling)
9382 // for the target we're vectorizing for, to make sure none of the
9383 // additional fp-math flags can help.
9384 if (Hints.isPotentiallyUnsafe() &&
9385 TTI->isFPVectorizationPotentiallyUnsafe()) {
9386 reportVectorizationFailure(
9387 "Potentially unsafe FP op prevents vectorization",
9388 "loop not vectorized due to unsafe FP support.",
9389 "UnsafeFP", ORE, L);
9390 Hints.emitRemarkWithHints();
9391 return false;
9392 }
9393
9394 bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
9395 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI());
9396
9397 // If an override option has been passed in for interleaved accesses, use it.
9398 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
9399 UseInterleaved = EnableInterleavedMemAccesses;
9400
9401 // Analyze interleaved memory accesses.
9402 if (UseInterleaved) {
9403 IAI.analyzeInterleaving(useMaskedInterleavedAccesses(*TTI));
9404 }
9405
9406 // Use the cost model.
9407 LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
9408 F, &Hints, IAI);
9409 CM.collectValuesToIgnore();
9410
9411 // Use the planner for vectorization.
9412 LoopVectorizationPlanner LVP(L, LI, TLI, TTI, &LVL, CM, IAI, PSE);
9413
9414 // Get user vectorization factor and interleave count.
9415 ElementCount UserVF = Hints.getWidth();
9416 unsigned UserIC = Hints.getInterleave();
9417
9418 // Plan how to best vectorize, return the best VF and its cost.
9419 Optional<VectorizationFactor> MaybeVF = LVP.plan(UserVF, UserIC);
9420
9421 VectorizationFactor VF = VectorizationFactor::Disabled();
9422 unsigned IC = 1;
9423
9424 if (MaybeVF) {
9425 VF = *MaybeVF;
9426 // Select the interleave count.
9427 IC = CM.selectInterleaveCount(VF.Width, VF.Cost);
9428 }
9429
9430 // Identify the diagnostic messages that should be produced.
9431 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
9432 bool VectorizeLoop = true, InterleaveLoop = true;
9433 if (Requirements.doesNotMeet(F, L, Hints)) {
9434 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: loop did not meet vectorization "
9435 "requirements.\n");
9436 Hints.emitRemarkWithHints();
9437 return false;
9438 }
9439
9440 if (VF.Width.isScalar()) {
9441 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
9442 VecDiagMsg = std::make_pair(
9443 "VectorizationNotBeneficial",
9444 "the cost-model indicates that vectorization is not beneficial");
9445 VectorizeLoop = false;
9446 }
9447
9448 if (!MaybeVF && UserIC > 1) {
9449 // Tell the user interleaving was avoided up-front, despite being explicitly
9450 // requested.
9451 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
9452 "interleaving should be avoided up front\n");
9453 IntDiagMsg = std::make_pair(
9454 "InterleavingAvoided",
9455 "Ignoring UserIC, because interleaving was avoided up front");
9456 InterleaveLoop = false;
9457 } else if (IC == 1 && UserIC <= 1) {
9458 // Tell the user interleaving is not beneficial.
9459 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
9460 IntDiagMsg = std::make_pair(
9461 "InterleavingNotBeneficial",
9462 "the cost-model indicates that interleaving is not beneficial");
9463 InterleaveLoop = false;
9464 if (UserIC == 1) {
9465 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
9466 IntDiagMsg.second +=
9467 " and is explicitly disabled or interleave count is set to 1";
9468 }
9469 } else if (IC > 1 && UserIC == 1) {
9470 // Tell the user interleaving is beneficial, but it explicitly disabled.
9471 LLVM_DEBUG(
9472 dbgs() << "LV: Interleaving is beneficial but is explicitly disabled.");
9473 IntDiagMsg = std::make_pair(
9474 "InterleavingBeneficialButDisabled",
9475 "the cost-model indicates that interleaving is beneficial "
9476 "but is explicitly disabled or interleave count is set to 1");
9477 InterleaveLoop = false;
9478 }
9479
9480 // Override IC if user provided an interleave count.
9481 IC = UserIC > 0 ? UserIC : IC;
9482
9483 // Emit diagnostic messages, if any.
9484 const char *VAPassName = Hints.vectorizeAnalysisPassName();
9485 if (!VectorizeLoop && !InterleaveLoop) {
9486 // Do not vectorize or interleaving the loop.
9487 ORE->emit([&]() {
9488 return OptimizationRemarkMissed(VAPassName, VecDiagMsg.first,
9489 L->getStartLoc(), L->getHeader())
9490 << VecDiagMsg.second;
9491 });
9492 ORE->emit([&]() {
9493 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
9494 L->getStartLoc(), L->getHeader())
9495 << IntDiagMsg.second;
9496 });
9497 return false;
9498 } else if (!VectorizeLoop && InterleaveLoop) {
9499 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
9500 ORE->emit([&]() {
9501 return OptimizationRemarkAnalysis(VAPassName, VecDiagMsg.first,
9502 L->getStartLoc(), L->getHeader())
9503 << VecDiagMsg.second;
9504 });
9505 } else if (VectorizeLoop && !InterleaveLoop) {
9506 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
9507 << ") in " << DebugLocStr << '\n');
9508 ORE->emit([&]() {
9509 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
9510 L->getStartLoc(), L->getHeader())
9511 << IntDiagMsg.second;
9512 });
9513 } else if (VectorizeLoop && InterleaveLoop) {
9514 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
9515 << ") in " << DebugLocStr << '\n');
9516 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
9517 }
9518
9519 LVP.setBestPlan(VF.Width, IC);
9520
9521 using namespace ore;
9522 bool DisableRuntimeUnroll = false;
9523 MDNode *OrigLoopID = L->getLoopID();
9524
9525 if (!VectorizeLoop) {
9526 assert(IC > 1 && "interleave count should not be 1 or 0");
9527 // If we decided that it is not legal to vectorize the loop, then
9528 // interleave it.
9529 InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL, &CM,
9530 BFI, PSI);
9531 LVP.executePlan(Unroller, DT);
9532
9533 ORE->emit([&]() {
9534 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
9535 L->getHeader())
9536 << "interleaved loop (interleaved count: "
9537 << NV("InterleaveCount", IC) << ")";
9538 });
9539 } else {
9540 // If we decided that it is *legal* to vectorize the loop, then do it.
9541
9542 // Consider vectorizing the epilogue too if it's profitable.
9543 VectorizationFactor EpilogueVF =
9544 CM.selectEpilogueVectorizationFactor(VF.Width, LVP);
9545 if (EpilogueVF.Width.isVector()) {
9546
9547 // The first pass vectorizes the main loop and creates a scalar epilogue
9548 // to be vectorized by executing the plan (potentially with a different
9549 // factor) again shortly afterwards.
9550 EpilogueLoopVectorizationInfo EPI(VF.Width.getKnownMinValue(), IC,
9551 EpilogueVF.Width.getKnownMinValue(), 1);
9552 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE, EPI,
9553 &LVL, &CM, BFI, PSI);
9554
9555 LVP.setBestPlan(EPI.MainLoopVF, EPI.MainLoopUF);
9556 LVP.executePlan(MainILV, DT);
9557 ++LoopsVectorized;
9558
9559 simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
9560 formLCSSARecursively(*L, *DT, LI, SE);
9561
9562 // Second pass vectorizes the epilogue and adjusts the control flow
9563 // edges from the first pass.
9564 LVP.setBestPlan(EPI.EpilogueVF, EPI.EpilogueUF);
9565 EPI.MainLoopVF = EPI.EpilogueVF;
9566 EPI.MainLoopUF = EPI.EpilogueUF;
9567 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC,
9568 ORE, EPI, &LVL, &CM, BFI, PSI);
9569 LVP.executePlan(EpilogILV, DT);
9570 ++LoopsEpilogueVectorized;
9571
9572 if (!MainILV.areSafetyChecksAdded())
9573 DisableRuntimeUnroll = true;
9574 } else {
9575 InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, IC,
9576 &LVL, &CM, BFI, PSI);
9577 LVP.executePlan(LB, DT);
9578 ++LoopsVectorized;
9579
9580 // Add metadata to disable runtime unrolling a scalar loop when there are
9581 // no runtime checks about strides and memory. A scalar loop that is
9582 // rarely used is not worth unrolling.
9583 if (!LB.areSafetyChecksAdded())
9584 DisableRuntimeUnroll = true;
9585 }
9586
9587 // Report the vectorization decision.
9588 ORE->emit([&]() {
9589 return OptimizationRemark(LV_NAME, "Vectorized", L->getStartLoc(),
9590 L->getHeader())
9591 << "vectorized loop (vectorization width: "
9592 << NV("VectorizationFactor", VF.Width)
9593 << ", interleaved count: " << NV("InterleaveCount", IC) << ")";
9594 });
9595 }
9596
9597 Optional<MDNode *> RemainderLoopID =
9598 makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
9599 LLVMLoopVectorizeFollowupEpilogue});
9600 if (RemainderLoopID.hasValue()) {
9601 L->setLoopID(RemainderLoopID.getValue());
9602 } else {
9603 if (DisableRuntimeUnroll)
9604 AddRuntimeUnrollDisableMetaData(L);
9605
9606 // Mark the loop as already vectorized to avoid vectorizing again.
9607 Hints.setAlreadyVectorized();
9608 }
9609
9610 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()));
9611 return true;
9612}
9613
9614LoopVectorizeResult LoopVectorizePass::runImpl(
9615 Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_,
9616 DominatorTree &DT_, BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_,
9617 DemandedBits &DB_, AAResults &AA_, AssumptionCache &AC_,
9618 std::function<const LoopAccessInfo &(Loop &)> &GetLAA_,
9619 OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_) {
9620 SE = &SE_;
9621 LI = &LI_;
9622 TTI = &TTI_;
9623 DT = &DT_;
9624 BFI = &BFI_;
9625 TLI = TLI_;
9626 AA = &AA_;
9627 AC = &AC_;
9628 GetLAA = &GetLAA_;
9629 DB = &DB_;
9630 ORE = &ORE_;
9631 PSI = PSI_;
9632
9633 // Don't attempt if
9634 // 1. the target claims to have no vector registers, and
9635 // 2. interleaving won't help ILP.
9636 //
9637 // The second condition is necessary because, even if the target has no
9638 // vector registers, loop vectorization may still enable scalar
9639 // interleaving.
9640 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) &&
9641 TTI->getMaxInterleaveFactor(1) < 2)
9642 return LoopVectorizeResult(false, false);
9643
9644 bool Changed = false, CFGChanged = false;
9645
9646 // The vectorizer requires loops to be in simplified form.
9647 // Since simplification may add new inner loops, it has to run before the
9648 // legality and profitability checks. This means running the loop vectorizer
9649 // will simplify all loops, regardless of whether anything end up being
9650 // vectorized.
9651 for (auto &L : *LI)
9652 Changed |= CFGChanged |=
9653 simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
9654
9655 // Build up a worklist of inner-loops to vectorize. This is necessary as
9656 // the act of vectorizing or partially unrolling a loop creates new loops
9657 // and can invalidate iterators across the loops.
9658 SmallVector<Loop *, 8> Worklist;
9659
9660 for (Loop *L : *LI)
9661 collectSupportedLoops(*L, LI, ORE, Worklist);
9662
9663 LoopsAnalyzed += Worklist.size();
9664
9665 // Now walk the identified inner loops.
9666 while (!Worklist.empty()) {
9667 Loop *L = Worklist.pop_back_val();
9668
9669 // For the inner loops we actually process, form LCSSA to simplify the
9670 // transform.
9671 Changed |= formLCSSARecursively(*L, *DT, LI, SE);
9672
9673 Changed |= CFGChanged |= processLoop(L);
9674 }
9675
9676 // Process each loop nest in the function.
9677 return LoopVectorizeResult(Changed, CFGChanged);
9678}
9679
9680PreservedAnalyses LoopVectorizePass::run(Function &F,
9681 FunctionAnalysisManager &AM) {
9682 auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
9683 auto &LI = AM.getResult<LoopAnalysis>(F);
9684 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
9685 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
9686 auto &BFI = AM.getResult<BlockFrequencyAnalysis>(F);
9687 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
9688 auto &AA = AM.getResult<AAManager>(F);
9689 auto &AC = AM.getResult<AssumptionAnalysis>(F);
9690 auto &DB = AM.getResult<DemandedBitsAnalysis>(F);
9691 auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
9692 MemorySSA *MSSA = EnableMSSALoopDependency
9693 ? &AM.getResult<MemorySSAAnalysis>(F).getMSSA()
9694 : nullptr;
9695
9696 auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
9697 std::function<const LoopAccessInfo &(Loop &)> GetLAA =
9698 [&](Loop &L) -> const LoopAccessInfo & {
9699 LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
9700 TLI, TTI, nullptr, MSSA};
9701 return LAM.getResult<LoopAccessAnalysis>(L, AR);
9702 };
9703 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
9704 ProfileSummaryInfo *PSI =
9705 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
9706 LoopVectorizeResult Result =
9707 runImpl(F, SE, LI, TTI, DT, BFI, &TLI, DB, AA, AC, GetLAA, ORE, PSI);
9708 if (!Result.MadeAnyChange)
9709 return PreservedAnalyses::all();
9710 PreservedAnalyses PA;
9711
9712 // We currently do not preserve loopinfo/dominator analyses with outer loop
9713 // vectorization. Until this is addressed, mark these analyses as preserved
9714 // only for non-VPlan-native path.
9715 // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
9716 if (!EnableVPlanNativePath) {
9717 PA.preserve<LoopAnalysis>();
9718 PA.preserve<DominatorTreeAnalysis>();
9719 }
9720 PA.preserve<BasicAA>();
9721 PA.preserve<GlobalsAA>();
9722 if (!Result.MadeCFGChange)
9723 PA.preserveSet<CFGAnalyses>();
9724 return PA;
9725}
9726